From 0647e9d9b7709750f7d3528d8e176ef37771234c Mon Sep 17 00:00:00 2001 From: Oleg Sheynin Date: Sun, 18 May 2025 21:43:53 -0400 Subject: [PATCH] fix --- VERSION | 2 +- .../market_data/_DEPRECATED_/crypto_md_day.sh | 44 ------- .../_DEPRECATED_/recorder/md_recorder.sh | 66 ---------- .../recorder/md_recorder_monitor.sh | 60 --------- docker/market_data/hist/load_alpaca_hbar.sh | 113 ----------------- docker/market_data/hist/load_alpaca_qat.sh | 115 ------------------ .../sim_prepare/prepare_eqt_simdata.sh | 1 + 7 files changed, 2 insertions(+), 399 deletions(-) delete mode 100755 docker/market_data/_DEPRECATED_/crypto_md_day.sh delete mode 100755 docker/market_data/_DEPRECATED_/recorder/md_recorder.sh delete mode 100755 docker/market_data/_DEPRECATED_/recorder/md_recorder_monitor.sh delete mode 100755 docker/market_data/hist/load_alpaca_hbar.sh delete mode 100755 docker/market_data/hist/load_alpaca_qat.sh diff --git a/VERSION b/VERSION index 29280fc..8785384 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.2.3,docker prepare sim - eqt - fix +2.2.4,docker prepare sim - eqt - fix diff --git a/docker/market_data/_DEPRECATED_/crypto_md_day.sh b/docker/market_data/_DEPRECATED_/crypto_md_day.sh deleted file mode 100755 index adcb0be..0000000 --- a/docker/market_data/_DEPRECATED_/crypto_md_day.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -usage() { - echo "Usage: ${0} []" - exit 1 -} - -# ------- Settings -DockerRegistry=cloud21.cvtt.vpn:5500 -# ------- Settings - -ImageTag=${1} -host=${2} -date=${3} - -if [ -z ${ImageTag} ] ; then - usage -fi - -if [ -z ${host} ] ; then - usage -fi - -if [ -z ${date} ] ; then - date=$(date -d "yesterday" +%Y%m%d) -fi - -DockerImage=${DockerRegistry}/crypto_md_day:${ImageTag} - - -Cmd="docker run" -Cmd+=" --pull=always" -Cmd+=" --network=host" -Cmd+=" --name=crypto_md_day.${host}.${date}" -Cmd+=" --rm" -Cmd+=" --volume=${HOME}/.creds:/.creds" -Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789" -Cmd+=" ${DockerImage}" -Cmd+=" -h ${host}" -Cmd+=" -d ${date}" -# Cmd+=" -s coinbase,bnbspot,bnbfut" -Cmd+=" -s coinbase,bnbspot" - -echo $Cmd -eval $Cmd diff --git a/docker/market_data/_DEPRECATED_/recorder/md_recorder.sh b/docker/market_data/_DEPRECATED_/recorder/md_recorder.sh deleted file mode 100755 index 0c49eb9..0000000 --- a/docker/market_data/_DEPRECATED_/recorder/md_recorder.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -ValidJobs=('BNBFUT_CLD21' 'BNBSPOT_CLD21' 'COINBASE_CLD21' 'BNBFUT_CVTTDATA' 'BNBSPOT_CVTTDATA' 'COINBASE_CVTTDATA') -# runs on host to start container -usage() { - echo "Usage: $0 [image_tag]" - exit 1 -} - -is_valid() { - local job=$1 - for valid_job in "${ValidJobs[@]}"; - do - # echo "job=$job valid_job=$valid_job" - if [[ "${job}" == "${valid_job}" ]]; then - return 0 - fi - done - return 1 -} - -is_container_running() { - local container_name=$1 - - if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then - return 0 # true - else - return 1 # false - fi -} - - -job=${1} -if ! is_valid "${job}"; then - usage -fi - -ImageTag=${2} -if [ "${ImageTag}" == "" ] ; then - ImageTag="1.6.9" -fi - -DockerImage=cloud21.cvtt.vpn:5500/md_recorder:${ImageTag} -ContainerName="md_recorder.${job}" - -if is_container_running "$ContainerName"; then - echo "Container ${ContainerName} is already running. Aborted." - exit -fi - - -Cmd="docker run" -Cmd+=" -d" -Cmd+=" --rm" -Cmd+=" --network=host" -# Cmd+=" --pull=always" -Cmd+=" --name=${ContainerName}" -Cmd+=" -v /home/cvtt/.creds:/.creds" -Cmd+=" -v /home/cvtt/prod/data:/app/data" -Cmd+=" -v /home/cvtt/prod/logs:/logs" -Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789" -Cmd+=" ${DockerImage}" -Cmd+=" ${job}" - -echo ${Cmd} -eval ${Cmd} diff --git a/docker/market_data/_DEPRECATED_/recorder/md_recorder_monitor.sh b/docker/market_data/_DEPRECATED_/recorder/md_recorder_monitor.sh deleted file mode 100755 index 59b93d7..0000000 --- a/docker/market_data/_DEPRECATED_/recorder/md_recorder_monitor.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -usage() { - echo "Usage: $0 []" - exit 1 -} - -ValidInstances=('CLD21' 'CVTTDATA') -is_valid() { - local inst=$1 - for valid_inst in "${ValidInstances[@]}"; - do - if [[ "$inst" == "$valid_inst" ]]; then - return 0 - fi - done - return 1 -} - -is_container_running() { - local container_name=$1 - - if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then - return 0 # true - else - return 1 # false - fi -} - -Instance=${1} -if ! is_valid "${Instance}"; then - usage -fi - -AdminPort=7225 -if [ "${2}" != "" ]; then - AdminPort=${2} -fi - -ContainerName="md_recorder_monitor.${Instance}" - -if is_container_running "$ContainerName"; then - echo "Container ${ContainerName} is already running. Aborted." - exit -fi - -Cmd="docker run" -Cmd+=" -d" -Cmd+=" --rm" -Cmd+=" --network=host" -Cmd+=" --pull=always" -Cmd+=" --name=${ContainerName}" -Cmd+=" -v /home/cvtt/.creds:/.creds" -Cmd+=" -v /home/cvtt/prod/logs:/logs" -Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789" -Cmd+=" cloud21.cvtt.vpn:5500/md_recorder_monitor:latest" -Cmd+=" ${Instance} ${AdminPort}" - -echo ${Cmd} -eval ${Cmd} diff --git a/docker/market_data/hist/load_alpaca_hbar.sh b/docker/market_data/hist/load_alpaca_hbar.sh deleted file mode 100755 index c6ba19d..0000000 --- a/docker/market_data/hist/load_alpaca_hbar.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/bin/bash - -usage() { - echo "Usage: $0 -N [-L ] [-d ] [-t ]" - exit 1 -} - -is_container_running() { - local container_name=$1 - - if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then - return 0 # true - else - return 1 # false - fi -} - -export CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS - -is_business_day() { - dt=${1} - - open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time') - if [ -n "${open_time}" ]; then - return 0 - else - return 1 - fi - -} -export -f is_business_day - -get_prev_business_day() { - Start=${1} - while true; do - if is_business_day ${Start}; then - break - fi - echo "${Start} is not business day in US" >&2 - Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d") - done - echo ${Start} -} -export -f get_prev_business_day - -# ----- Settings -DockerRegistry=cloud21.cvtt.vpn:5500 -ContainerName=alpaca_hbar -LogDir=/home/cvtt/prod/logs -ImageTag=0.1.7 -# ----- Settings - -while getopts ":d:N:L:t:" opt; do - case ${opt} in - d ) - date_to_load=$OPTARG - ;; - N ) - NumSymbols=$OPTARG - ;; - L ) - LogDir=$OPTARG - ;; - t ) - ImageTag=$OPTARG - ;; - \? ) - echo "Invalid option: -$OPTARG" >&2 - usage - ;; - : ) - echo "Option -$OPTARG requires an argument." >&2 - usage - ;; - esac -done - -DockerImage=${DockerRegistry}/alpaca_hbar:${ImageTag} #:latest - - -if is_container_running "$ContainerName"; then - echo "Container ${ContainerName} is already running." - exit 3 -fi - -if [ -z "${date_to_load}" ]; then - echo "date_to_load is empty" - date_to_load=$(get_prev_business_day $(date -d "yesterday" '+%Y-%m-%d')) - echo "Historical Data for ${date_to_load}" -fi - -echo "date_to_load=${date_to_load}" - -Cmd="docker run" -Cmd+=" --pull=always" -Cmd+=" --network=host" -Cmd+=" --name=${ContainerName}" -Cmd+=" --rm" -Cmd+=" --volume=${LogDir}:/logs" -Cmd+=" --volume=${HOME}/.creds:/.creds" -Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789" -Cmd+=" ${DockerImage}" -Cmd+=" -d ${date_to_load}" -if [ -n "${NumSymbols}" ]; then - Cmd+=" -N ${NumSymbols}" -fi - -echo $Cmd -eval $Cmd - -if [ "$?" != "0" ] ; then - exit 1 # if killed we do not save last day -fi diff --git a/docker/market_data/hist/load_alpaca_qat.sh b/docker/market_data/hist/load_alpaca_qat.sh deleted file mode 100755 index 503946c..0000000 --- a/docker/market_data/hist/load_alpaca_qat.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash - -usage() { - echo "Usage: $0 -S [-L ] [-d ] [-t ]" - exit 1 -} - -is_container_running() { - local container_name=$1 - - if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then - return 0 # true - else - return 1 # false - fi -} - -export CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS - -is_business_day() { - dt=${1} - - open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time') - if [ -n "${open_time}" ]; then - return 0 - else - return 1 - fi - -} -export -f is_business_day - -get_prev_business_day() { - Start=${1} - while true; do - if is_business_day ${Start}; then - break - fi - echo "${Start} is not business day in US" >&2 - Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d") - done - echo ${Start} -} -export -f get_prev_business_day - -# ----- Settings -DockerRegistry=cloud21.cvtt.vpn:5500 -ContainerName=alpaca_qat -LogDir=/home/cvtt/prod/logs -ImageTag=0.1.7 -# ----- Settings - -while getopts ":d:S:L:t:" opt; do - case ${opt} in - d ) - date_to_load=$OPTARG - ;; - S ) - Symbols=$OPTARG - ;; - L ) - LogDir=$OPTARG - ;; - t ) - ImageTag=$OPTARG - ;; - \? ) - echo "Invalid option: -$OPTARG" >&2 - usage - ;; - : ) - echo "Option -$OPTARG requires an argument." >&2 - usage - ;; - esac -done - -if [ -z ${Symbols} ] ; then - echo "symbols are not specified" - usage -fi - -if is_container_running "$ContainerName"; then - echo "Container ${ContainerName} is already running." - exit 3 -fi - -DockerImage=${DockerRegistry}/alpaca_qat:${ImageTag} #:latest - -if [ -z "${date_to_load}" ]; then - echo "date_to_load is empty" - date_to_load=$(get_prev_business_day $(date -d "yesterday" '+%Y-%m-%d')) - echo "Historical Data for ${date_to_load}" -fi - -echo "date_to_load=${date_to_load}" - -Cmd="docker run" -Cmd+=" --pull=always" -Cmd+=" --network=host" -Cmd+=" --name=${ContainerName}" -Cmd+=" --rm" -Cmd+=" --volume=${LogDir}:/logs" -Cmd+=" --volume=${HOME}/.creds:/.creds" -Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789" -Cmd+=" ${DockerImage}" -Cmd+=" -d ${date_to_load}" -Cmd+=" -S ${Symbols}" - -echo $Cmd -eval $Cmd - -if [ "$?" != "0" ] ; then - exit 1 # if killed we do not save last day -fi diff --git a/scripts/mkt_data/sim_prepare/prepare_eqt_simdata.sh b/scripts/mkt_data/sim_prepare/prepare_eqt_simdata.sh index f1383c4..782bd59 100755 --- a/scripts/mkt_data/sim_prepare/prepare_eqt_simdata.sh +++ b/scripts/mkt_data/sim_prepare/prepare_eqt_simdata.sh @@ -64,6 +64,7 @@ for sym in ${Symbols[@]}; do for src_file in ${SourceHbarFile} ${SourceQatFile}; do Cmd="rsync -ahv" + Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'" Cmd+=" ${SOURCE_HOST}:${src_file}" Cmd+=" $OUTPUT_DIR/" echo ${Cmd}