progress

healthcheck/md/crypto_missing_dates.sh

@@ -1,11 +1,5 @@
 #!/bin/bash
 
-SourceHost=${1}
-if [ "${SourceHost}" == "" ]; then
-   echo "Usage: ${0} <source_host>"
-   exit 1
-fi
-
 function_file=$(realpath $(dirname $0))/get_retrofit_dates.sh
 source ${function_file}
 
@@ -15,8 +9,23 @@ Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn}
 FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
 BackDaysCount=${BACKDAYS_COUNT:-30}
 
+for SourceHost in cloud28 cloud29; do
+    echo "============= ${SourceHost}"
+    Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/crypto/${SourceHost} ${FileGlob} ${BackDaysCount}"
+    echo $Cmd
 
-Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/crypto/${SourceHost} ${FileGlob} ${BackDaysCount}"
+    DATES=$($Cmd)
+
+    for dt in ${DATES}; do
+        echo $dt
+    done
+done
+
+DEFAULT_FGLOB=*.crypto_sim_md.db.gz
+FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
+
+echo "============= sim"
+Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/crypto/sim ${FileGlob} ${BackDaysCount}"
 echo $Cmd
 
 DATES=$($Cmd)

healthcheck/md/crypto_sim_missing_dates.sh

@@ -1,22 +0,0 @@
-#!/bin/bash
-
-# DAILY_SIM_CRYPTO_DATE=20250725 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up daily_sim_crypto 
-#
-#
-source $(realpath $(dirname $0))/get_retrofit_dates.sh
-
-DEFAULT_FGLOB=*.crypto_sim_md.db.gz
-
-Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn}
-FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
-BackDaysCount=${BACKDAYS_COUNT:-30}
-
-Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/crypto/sim ${FileGlob} ${BackDaysCount}"
-echo $Cmd
-
-DATES=$($Cmd)
-
-for dt in ${DATES}; do
-   echo $dt
-done
-

healthcheck/md/equity_missing_dates.sh

@@ -11,6 +11,7 @@ Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn}
 FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
 BackDaysCount=${BACKDAYS_COUNT:-50}
 
+echo "======================= Alpaca MD"
 Cmd="get_equity_retrofit_dates  ${Source} /works/cvtt/md_archive/equity/alpaca_md/2025/N/NVDA ${FileGlob} ${BackDaysCount}"
 echo $Cmd
 
@@ -19,3 +20,26 @@ DATES=$($Cmd)
 for dt in ${DATES}; do
    echo $dt
 done
+# ======================================
+
+DEFAULT_FGLOB=*.alpaca_sim_md.db.gz
+FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
+
+echo "======================= sim"
+Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/equity/alpaca_md/sim ${FileGlob} ${BackDaysCount}"
+echo $Cmd
+
+DATES=$($Cmd)
+
+for dt in ${DATES}; do
+    weekday=$(date -d "$dt" +%u)
+    (( weekday >= 6 )) && continue
+    is_equity_business_date "$dt"
+    rc=$?
+    if (( rc != 0 )); then
+        continue
+    fi
+     
+    fdt=$(date -d "$dt" +%Y-%m-%d 2>/dev/null)
+    echo $fdt
+done

healthcheck/md/equity_sim_missing_dates.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-
-#
-# DAILY_SIM_EQUITY_DATE=20250723 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up daily_sim_equity 
-#
-source $(realpath $(dirname $0))/get_retrofit_dates.sh
-
-DEFAULT_FGLOB=*.alpaca_sim_md.db.gz
-
-Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn}
-FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
-BackDaysCount=${BACKDAYS_COUNT:-50}
-
-Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/equity/alpaca_md/sim ${FileGlob} ${BackDaysCount}"
-echo $Cmd
-
-DATES=$($Cmd)
-
-for dt in ${DATES}; do
-      weekday=$(date -d "$dt" +%u)
-      (( weekday >= 6 )) && continue
-
-      fdt=$(date -d "$dt" +%Y-%m-%d 2>/dev/null)
-      echo $fdt
-done

healthcheck/md/get_retrofit_dates.sh

@@ -61,6 +61,30 @@ get_retrofit_dates() {
     printf '%s\n' "${missing_dates[@]}"
 }
 
+is_equity_business_date() {
+    local ymd=${1:-}
+
+    if [[ ! "$ymd" =~ ^[0-9]{8}$ ]]; then
+        echo "is_equity_business_date: expected YYYYMMDD, got \"$ymd\"" >&2
+        return 2
+    fi
+
+    local iso_date="${ymd:0:4}-${ymd:4:2}-${ymd:6:2}"
+    local url="https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS&start=${iso_date}&end=${iso_date}"
+    local response
+
+    if ! response=$(curl -fsS "$url" 2>/dev/null); then
+        echo "is_equity_business_date: failed to query trading calendar API" >&2
+        return 1
+    fi
+
+    if [[ "$response" =~ \[\s*\] ]]; then
+        return 1
+    fi
+
+    return 0
+}
+
 get_equity_retrofit_dates() {
     local host=${1:-}
     local root_dir=${2:-}
@@ -107,10 +131,15 @@ get_equity_retrofit_dates() {
     fi
 
     local -a missing_dates=()
-    local idx dt tmp
+    local idx dt tmp rc
     for (( idx = ${#target_dates[@]} - 1; idx >= 0; idx-- )); do
         dt=${target_dates[$idx]}
         [[ -n "${existing_dates[$dt]:-}" ]] && continue
+        is_equity_business_date "$dt"
+        rc=$?
+        if (( rc != 0 )); then
+            continue
+        fi
         if ! tmp=$(date -d "$dt" +%Y-%m-%d 2>/dev/null); then
             echo "invalid date \"$dt\"" >&2
             return 1

healthcheck/md/md_checklist.sh

@@ -31,12 +31,8 @@ run_crypto_checklist() {
 
     declare -A Commands
     Commands=(
-        ["crypto:missing-dates:hs01:cloud28"]="MD_HOST=hs01.cvtt.vpn ${SDir}/crypto_missing_dates.sh cloud28"
+        ["crypto:missing-dates:cloud21"]="MD_HOST=cloud21.cvtt.vpn ${SDir}/crypto_missing_dates.sh"
-        ["crypto:missing-dates:hs01:cloud29"]="MD_HOST=hs01.cvtt.vpn ${SDir}/crypto_missing_dates.sh cloud29"
+        ["crypto:missing-dates:hs01"]="MD_HOST=hs01.cvtt.vpn ${SDir}/crypto_missing_dates.sh"
-        ["crypto:missing-dates:hs01:sim"]="MD_HOST=hs01.cvtt.vpn ${SDir}/crypto_sim_missing_dates.sh"
-        ["crypto:missing-dates:cloud21:cloud28"]="${SDir}/crypto_missing_dates.sh cloud28"
-        ["crypto:missing-dates:cloud21:cloud29"]="${SDir}/crypto_missing_dates.sh cloud29"
-        ["crypto:missing-dates:cloud21:sim"]="${SDir}/crypto_sim_missing_dates.sh"
     )
 
     for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
@@ -61,10 +57,8 @@ run_eqt_checklist() {
 
     declare -A Commands
     Commands=(
-        ["equity:missing-dates:cloud21"]="${SDir}/equity_missing_dates.sh"
+        ["equity:missing-dates:cloud21"]="MD_HOST=cloud21.cvtt.vpn ${SDir}/equity_missing_dates.sh"
-        ["equity:missing-dates:cloud21:sim"]="${SDir}/equity_sim_missing_dates.sh"
         ["equity:missing-dates:hs01"]="MD_HOST=hs01.cvtt.vpn ${SDir}/equity_missing_dates.sh"
-        ["equity:missing-dates:hs01:sim"]="MD_HOST=hs01.cvtt.vpn ${SDir}/equity_sim_missing_dates.sh"
     )    
 
     for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
@@ -91,6 +85,9 @@ run_crypto_checklist  >> ${tmpfile}
 echo "Equity checklist..."
 run_eqt_checklist >> ${tmpfile}
 
+if tty -s; then
+    cat ${tmpfile}
+fi
 cat ${tmpfile} |  ${Sender} ${StatusChannel}
 
 echo "Done $0"