From b3fb3e01b306311cc0841a7a8a5180237867e046 Mon Sep 17 00:00:00 2001 From: Cryptoval Trading Technologies Date: Fri, 28 Nov 2025 19:45:31 +0000 Subject: [PATCH] progress with retrofit --- .env | 19 +++++++---------- .gitignore | 1 + retrofit/crypto_missing_dates.sh | 24 ++++++++++++++------- retrofit/crypto_sim_missing_dates.sh | 22 -------------------- retrofit/equity_missing_dates.sh | 24 +++++++++++++++++++++ retrofit/equity_sim_missing_dates.sh | 25 ---------------------- retrofit/get_retrofit_dates.sh | 31 +++++++++++++++++++++++++++- 7 files changed, 80 insertions(+), 66 deletions(-) delete mode 100755 retrofit/crypto_sim_missing_dates.sh delete mode 100755 retrofit/equity_sim_missing_dates.sh diff --git a/.env b/.env index 252c415..33d0105 100644 --- a/.env +++ b/.env @@ -1,7 +1,7 @@ CREDS_FILE=/config/.creds # ==================================== daily_crypto_md -CRYPTO_IMAGE_VERSION=0.0.4 +CRYPTO_IMAGE_VERSION=0.0.6 CRYPTO_CONFIG_FILE=http://cloud16.cvtt.vpn:6789/apps/utils/mktdata/daily_md_crypto CRYPTO_CONTAINER_NAME_PFX=daily_crypto_md CRYPTO_DB_SCHEMAS=coinbase,bnbspot @@ -9,7 +9,7 @@ CRYPTO_DB_SCHEMAS=coinbase,bnbspot CRYPTO_MD_DATE= CRYPTO_RSYNC_TARGETS= -CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto" +CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/works/cvtt/md_archive/crypto" CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto" CRYPTO_ADD_ARGS="--log_stdout" @@ -20,7 +20,7 @@ CRYPTO_PYTHON_DEBUG_WAIT=Y CRYPTO_PYTHON_DEBUG_PORT=45678 # ==================================== daily_equity_md -EQUITY_IMAGE_VERSION=0.0.2 +EQUITY_IMAGE_VERSION=0.0.7 EQUITY_CONTAINER_NAME_PFX=daily_equity_md EQUITY_HBAR_NUM_SYMBOLS= EQUITY_QAT_SYMBOLS=COIN,GBTC,HOOD,MSTR,PYPL,QQQ,NVDA @@ -28,22 +28,19 @@ EQUITY_QAT_SYMBOLS=COIN,GBTC,HOOD,MSTR,PYPL,QQQ,NVDA EQUITY_HBAR_DATE= EQUITY_RSYNC_TARGETS= -EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity" -EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity" -# EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/test" +EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md" +EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md" # ==================================== daily_sim_crypto SIM_CRYPTO_IMAGE_VERSION=0.0.3 DAILY_SIM_CRYPTO_DATE= -#DAILY_SIM_CRYPTO_SOURCE_HOST=cloud21.cvtt.vpn -#DAILY_SIM_CRYPTO_SOURCE_DIR=/opt/store/cvtt/md_archive/crypto DAILY_SIM_CRYPTO_SOURCE_HOST=hs01.cvtt.vpn DAILY_SIM_CRYPTO_SOURCE_DIR=/works/cvtt/md_archive/crypto DAILY_SIM_CRYPTO_DB_SOURCES=cloud28,cloud29 DAILY_SIM_CRYPTO_RSYNC_TARGETS= DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/sim" -DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim" +DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/works/cvtt/md_archive/crypto/sim" # ==================================== daily_sim_equity @@ -51,10 +48,10 @@ SIM_EQUITY_IMAGE_VERSION=0.0.2 DAILY_SIM_EQUITY_DATE= DAILY_SIM_EQUITY_SOURCE_HOST=cloud21.cvtt.vpn -DAILY_SIM_EQUITY_SOURCE_DIR=/opt/store/cvtt/md_archive/equity/alpaca_md +DAILY_SIM_EQUITY_SOURCE_DIR=/works/cvtt/md_archive/equity/alpaca_md DAILY_SIM_EQUITY_SYMBOL_LIST=COIN #,CRCL,GBTC,HOOD,MSTR DAILY_SIM_EQUITY_RSYNC_TARGETS= DAILY_SIM_EQUITY_RSYNC_TARGETS="${DAILY_SIM_EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim" -DAILY_SIM_EQUITY_RSYNC_TARGETS="${DAILY_SIM_EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim" +DAILY_SIM_EQUITY_RSYNC_TARGETS="${DAILY_SIM_EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim" diff --git a/.gitignore b/.gitignore index c66e0c4..9548569 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ logs/ nohup.out .history/ __OLD__/ +__DELETE__/ \ No newline at end of file diff --git a/retrofit/crypto_missing_dates.sh b/retrofit/crypto_missing_dates.sh index 60cf446..fc23a9d 100755 --- a/retrofit/crypto_missing_dates.sh +++ b/retrofit/crypto_missing_dates.sh @@ -1,11 +1,5 @@ #!/bin/bash -SourceHost=${1} -if [ "${SourceHost}" == "" ]; then - echo "Usage: ${0} " - exit 1 -fi - function_file=$(realpath $(dirname $0))/get_retrofit_dates.sh source ${function_file} @@ -15,8 +9,23 @@ Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn} FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}} BackDaysCount=${BACKDAYS_COUNT:-30} +for SourceHost in cloud28 cloud29; do + echo "============= ${SourceHost}" + Cmd="get_retrofit_dates ${Source} /works/cvtt/md_archive/crypto/${SourceHost} ${FileGlob} ${BackDaysCount}" + echo $Cmd -Cmd="get_retrofit_dates ${Source} /works/cvtt/md_archive/crypto/${SourceHost} ${FileGlob} ${BackDaysCount}" + DATES=$($Cmd) + + for dt in ${DATES}; do + echo $dt + done +done + +DEFAULT_FGLOB=*.crypto_sim_md.db.gz +FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}} + +echo "============= sim" +Cmd="get_retrofit_dates ${Source} /works/cvtt/md_archive/crypto/sim ${FileGlob} ${BackDaysCount}" echo $Cmd DATES=$($Cmd) @@ -24,3 +33,4 @@ DATES=$($Cmd) for dt in ${DATES}; do echo $dt done + diff --git a/retrofit/crypto_sim_missing_dates.sh b/retrofit/crypto_sim_missing_dates.sh deleted file mode 100755 index ce26a3e..0000000 --- a/retrofit/crypto_sim_missing_dates.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# DAILY_SIM_CRYPTO_DATE=20250725 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up daily_sim_crypto -# -# -source $(realpath $(dirname $0))/get_retrofit_dates.sh - -DEFAULT_FGLOB=*.crypto_sim_md.db.gz - -Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn} -FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}} -BackDaysCount=${BACKDAYS_COUNT:-30} - -Cmd="get_retrofit_dates ${Source} /works/cvtt/md_archive/crypto/sim ${FileGlob} ${BackDaysCount}" -echo $Cmd - -DATES=$($Cmd) - -for dt in ${DATES}; do - echo $dt -done - diff --git a/retrofit/equity_missing_dates.sh b/retrofit/equity_missing_dates.sh index 79cf9f2..de8d0ee 100755 --- a/retrofit/equity_missing_dates.sh +++ b/retrofit/equity_missing_dates.sh @@ -11,6 +11,7 @@ Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn} FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}} BackDaysCount=${BACKDAYS_COUNT:-50} +echo "======================= Alpaca MD" Cmd="get_equity_retrofit_dates ${Source} /works/cvtt/md_archive/equity/alpaca_md/2025/N/NVDA ${FileGlob} ${BackDaysCount}" echo $Cmd @@ -19,3 +20,26 @@ DATES=$($Cmd) for dt in ${DATES}; do echo $dt done +# ====================================== + +DEFAULT_FGLOB=*.alpaca_sim_md.db.gz +FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}} + +echo "======================= sim" +Cmd="get_retrofit_dates ${Source} /works/cvtt/md_archive/equity/alpaca_md/sim ${FileGlob} ${BackDaysCount}" +echo $Cmd + +DATES=$($Cmd) + +for dt in ${DATES}; do + weekday=$(date -d "$dt" +%u) + (( weekday >= 6 )) && continue + is_equity_business_date "$dt" + rc=$? + if (( rc != 0 )); then + continue + fi + + fdt=$(date -d "$dt" +%Y-%m-%d 2>/dev/null) + echo $fdt +done diff --git a/retrofit/equity_sim_missing_dates.sh b/retrofit/equity_sim_missing_dates.sh deleted file mode 100755 index b573c1e..0000000 --- a/retrofit/equity_sim_missing_dates.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# -# DAILY_SIM_EQUITY_DATE=20250723 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up daily_sim_equity -# -source $(realpath $(dirname $0))/get_retrofit_dates.sh - -DEFAULT_FGLOB=*.alpaca_sim_md.db.gz - -Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn} -FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}} -BackDaysCount=${BACKDAYS_COUNT:-50} - -Cmd="get_retrofit_dates ${Source} /works/cvtt/md_archive/equity/alpaca_md/sim ${FileGlob} ${BackDaysCount}" -echo $Cmd - -DATES=$($Cmd) - -for dt in ${DATES}; do - weekday=$(date -d "$dt" +%u) - (( weekday >= 6 )) && continue - - fdt=$(date -d "$dt" +%Y-%m-%d 2>/dev/null) - echo $fdt -done diff --git a/retrofit/get_retrofit_dates.sh b/retrofit/get_retrofit_dates.sh index cf3452b..a5d055f 100755 --- a/retrofit/get_retrofit_dates.sh +++ b/retrofit/get_retrofit_dates.sh @@ -61,6 +61,30 @@ get_retrofit_dates() { printf '%s\n' "${missing_dates[@]}" } +is_equity_business_date() { + local ymd=${1:-} + + if [[ ! "$ymd" =~ ^[0-9]{8}$ ]]; then + echo "is_equity_business_date: expected YYYYMMDD, got \"$ymd\"" >&2 + return 2 + fi + + local iso_date="${ymd:0:4}-${ymd:4:2}-${ymd:6:2}" + local url="https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS&start=${iso_date}&end=${iso_date}" + local response + + if ! response=$(curl -fsS "$url" 2>/dev/null); then + echo "is_equity_business_date: failed to query trading calendar API" >&2 + return 1 + fi + + if [[ "$response" =~ \[\s*\] ]]; then + return 1 + fi + + return 0 +} + get_equity_retrofit_dates() { local host=${1:-} local root_dir=${2:-} @@ -107,10 +131,15 @@ get_equity_retrofit_dates() { fi local -a missing_dates=() - local idx dt tmp + local idx dt tmp rc for (( idx = ${#target_dates[@]} - 1; idx >= 0; idx-- )); do dt=${target_dates[$idx]} [[ -n "${existing_dates[$dt]:-}" ]] && continue + is_equity_business_date "$dt" + rc=$? + if (( rc != 0 )); then + continue + fi if ! tmp=$(date -d "$dt" +%Y-%m-%d 2>/dev/null); then echo "invalid date \"$dt\"" >&2 return 1