From 0f34dd7827076bfd5204f3693d6e0e5d26c0f868 Mon Sep 17 00:00:00 2001 From: Cryptoval Trading Technologies Date: Tue, 6 Jan 2026 15:08:38 +0000 Subject: [PATCH] progress --- healthcheck/md/crypto_missing_dates.sh | 25 +++++++++++------ healthcheck/md/crypto_sim_missing_dates.sh | 22 --------------- healthcheck/md/equity_missing_dates.sh | 24 +++++++++++++++++ healthcheck/md/equity_sim_missing_dates.sh | 25 ----------------- healthcheck/md/get_retrofit_dates.sh | 31 +++++++++++++++++++++- healthcheck/md/md_checklist.sh | 15 +++++------ 6 files changed, 77 insertions(+), 65 deletions(-) delete mode 100755 healthcheck/md/crypto_sim_missing_dates.sh delete mode 100755 healthcheck/md/equity_sim_missing_dates.sh diff --git a/healthcheck/md/crypto_missing_dates.sh b/healthcheck/md/crypto_missing_dates.sh index 60cf446..9f08500 100755 --- a/healthcheck/md/crypto_missing_dates.sh +++ b/healthcheck/md/crypto_missing_dates.sh @@ -1,11 +1,5 @@ #!/bin/bash -SourceHost=${1} -if [ "${SourceHost}" == "" ]; then - echo "Usage: ${0} " - exit 1 -fi - function_file=$(realpath $(dirname $0))/get_retrofit_dates.sh source ${function_file} @@ -15,12 +9,27 @@ Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn} FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}} BackDaysCount=${BACKDAYS_COUNT:-30} +for SourceHost in cloud28 cloud29; do + echo "============= ${SourceHost}" + Cmd="get_retrofit_dates ${Source} /works/cvtt/md_archive/crypto/${SourceHost} ${FileGlob} ${BackDaysCount}" + echo $Cmd -Cmd="get_retrofit_dates ${Source} /works/cvtt/md_archive/crypto/${SourceHost} ${FileGlob} ${BackDaysCount}" + DATES=$($Cmd) + + for dt in ${DATES}; do + echo $dt + done +done + +DEFAULT_FGLOB=*.crypto_sim_md.db.gz +FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}} + +echo "============= sim" +Cmd="get_retrofit_dates ${Source} /works/cvtt/md_archive/crypto/sim ${FileGlob} ${BackDaysCount}" echo $Cmd DATES=$($Cmd) for dt in ${DATES}; do echo $dt -done +done \ No newline at end of file diff --git a/healthcheck/md/crypto_sim_missing_dates.sh b/healthcheck/md/crypto_sim_missing_dates.sh deleted file mode 100755 index ce26a3e..0000000 --- a/healthcheck/md/crypto_sim_missing_dates.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# DAILY_SIM_CRYPTO_DATE=20250725 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up daily_sim_crypto -# -# -source $(realpath $(dirname $0))/get_retrofit_dates.sh - -DEFAULT_FGLOB=*.crypto_sim_md.db.gz - -Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn} -FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}} -BackDaysCount=${BACKDAYS_COUNT:-30} - -Cmd="get_retrofit_dates ${Source} /works/cvtt/md_archive/crypto/sim ${FileGlob} ${BackDaysCount}" -echo $Cmd - -DATES=$($Cmd) - -for dt in ${DATES}; do - echo $dt -done - diff --git a/healthcheck/md/equity_missing_dates.sh b/healthcheck/md/equity_missing_dates.sh index 79cf9f2..de8d0ee 100755 --- a/healthcheck/md/equity_missing_dates.sh +++ b/healthcheck/md/equity_missing_dates.sh @@ -11,6 +11,7 @@ Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn} FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}} BackDaysCount=${BACKDAYS_COUNT:-50} +echo "======================= Alpaca MD" Cmd="get_equity_retrofit_dates ${Source} /works/cvtt/md_archive/equity/alpaca_md/2025/N/NVDA ${FileGlob} ${BackDaysCount}" echo $Cmd @@ -19,3 +20,26 @@ DATES=$($Cmd) for dt in ${DATES}; do echo $dt done +# ====================================== + +DEFAULT_FGLOB=*.alpaca_sim_md.db.gz +FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}} + +echo "======================= sim" +Cmd="get_retrofit_dates ${Source} /works/cvtt/md_archive/equity/alpaca_md/sim ${FileGlob} ${BackDaysCount}" +echo $Cmd + +DATES=$($Cmd) + +for dt in ${DATES}; do + weekday=$(date -d "$dt" +%u) + (( weekday >= 6 )) && continue + is_equity_business_date "$dt" + rc=$? + if (( rc != 0 )); then + continue + fi + + fdt=$(date -d "$dt" +%Y-%m-%d 2>/dev/null) + echo $fdt +done diff --git a/healthcheck/md/equity_sim_missing_dates.sh b/healthcheck/md/equity_sim_missing_dates.sh deleted file mode 100755 index b573c1e..0000000 --- a/healthcheck/md/equity_sim_missing_dates.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# -# DAILY_SIM_EQUITY_DATE=20250723 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up daily_sim_equity -# -source $(realpath $(dirname $0))/get_retrofit_dates.sh - -DEFAULT_FGLOB=*.alpaca_sim_md.db.gz - -Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn} -FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}} -BackDaysCount=${BACKDAYS_COUNT:-50} - -Cmd="get_retrofit_dates ${Source} /works/cvtt/md_archive/equity/alpaca_md/sim ${FileGlob} ${BackDaysCount}" -echo $Cmd - -DATES=$($Cmd) - -for dt in ${DATES}; do - weekday=$(date -d "$dt" +%u) - (( weekday >= 6 )) && continue - - fdt=$(date -d "$dt" +%Y-%m-%d 2>/dev/null) - echo $fdt -done diff --git a/healthcheck/md/get_retrofit_dates.sh b/healthcheck/md/get_retrofit_dates.sh index cf3452b..a5d055f 100755 --- a/healthcheck/md/get_retrofit_dates.sh +++ b/healthcheck/md/get_retrofit_dates.sh @@ -61,6 +61,30 @@ get_retrofit_dates() { printf '%s\n' "${missing_dates[@]}" } +is_equity_business_date() { + local ymd=${1:-} + + if [[ ! "$ymd" =~ ^[0-9]{8}$ ]]; then + echo "is_equity_business_date: expected YYYYMMDD, got \"$ymd\"" >&2 + return 2 + fi + + local iso_date="${ymd:0:4}-${ymd:4:2}-${ymd:6:2}" + local url="https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS&start=${iso_date}&end=${iso_date}" + local response + + if ! response=$(curl -fsS "$url" 2>/dev/null); then + echo "is_equity_business_date: failed to query trading calendar API" >&2 + return 1 + fi + + if [[ "$response" =~ \[\s*\] ]]; then + return 1 + fi + + return 0 +} + get_equity_retrofit_dates() { local host=${1:-} local root_dir=${2:-} @@ -107,10 +131,15 @@ get_equity_retrofit_dates() { fi local -a missing_dates=() - local idx dt tmp + local idx dt tmp rc for (( idx = ${#target_dates[@]} - 1; idx >= 0; idx-- )); do dt=${target_dates[$idx]} [[ -n "${existing_dates[$dt]:-}" ]] && continue + is_equity_business_date "$dt" + rc=$? + if (( rc != 0 )); then + continue + fi if ! tmp=$(date -d "$dt" +%Y-%m-%d 2>/dev/null); then echo "invalid date \"$dt\"" >&2 return 1 diff --git a/healthcheck/md/md_checklist.sh b/healthcheck/md/md_checklist.sh index 30b7a93..7b49c50 100755 --- a/healthcheck/md/md_checklist.sh +++ b/healthcheck/md/md_checklist.sh @@ -31,12 +31,8 @@ run_crypto_checklist() { declare -A Commands Commands=( - ["crypto:missing-dates:hs01:cloud28"]="MD_HOST=hs01.cvtt.vpn ${SDir}/crypto_missing_dates.sh cloud28" - ["crypto:missing-dates:hs01:cloud29"]="MD_HOST=hs01.cvtt.vpn ${SDir}/crypto_missing_dates.sh cloud29" - ["crypto:missing-dates:hs01:sim"]="MD_HOST=hs01.cvtt.vpn ${SDir}/crypto_sim_missing_dates.sh" - ["crypto:missing-dates:cloud21:cloud28"]="${SDir}/crypto_missing_dates.sh cloud28" - ["crypto:missing-dates:cloud21:cloud29"]="${SDir}/crypto_missing_dates.sh cloud29" - ["crypto:missing-dates:cloud21:sim"]="${SDir}/crypto_sim_missing_dates.sh" + ["crypto:missing-dates:cloud21"]="MD_HOST=cloud21.cvtt.vpn ${SDir}/crypto_missing_dates.sh" + ["crypto:missing-dates:hs01"]="MD_HOST=hs01.cvtt.vpn ${SDir}/crypto_missing_dates.sh" ) for name in $(printf "%s\n" "${!Commands[@]}" | sort); do @@ -61,10 +57,8 @@ run_eqt_checklist() { declare -A Commands Commands=( - ["equity:missing-dates:cloud21"]="${SDir}/equity_missing_dates.sh" - ["equity:missing-dates:cloud21:sim"]="${SDir}/equity_sim_missing_dates.sh" + ["equity:missing-dates:cloud21"]="MD_HOST=cloud21.cvtt.vpn ${SDir}/equity_missing_dates.sh" ["equity:missing-dates:hs01"]="MD_HOST=hs01.cvtt.vpn ${SDir}/equity_missing_dates.sh" - ["equity:missing-dates:hs01:sim"]="MD_HOST=hs01.cvtt.vpn ${SDir}/equity_sim_missing_dates.sh" ) for name in $(printf "%s\n" "${!Commands[@]}" | sort); do @@ -91,6 +85,9 @@ run_crypto_checklist >> ${tmpfile} echo "Equity checklist..." run_eqt_checklist >> ${tmpfile} +if tty -s; then + cat ${tmpfile} +fi cat ${tmpfile} | ${Sender} ${StatusChannel} echo "Done $0"