progress with retrofit

retrofit/crypto_missing_dates.sh

@@ -1,11 +1,5 @@
 #!/bin/bash
 
-SourceHost=${1}
-if [ "${SourceHost}" == "" ]; then
-   echo "Usage: ${0} <source_host>"
-   exit 1
-fi
-
 function_file=$(realpath $(dirname $0))/get_retrofit_dates.sh
 source ${function_file}
 
@@ -15,8 +9,23 @@ Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn}
 FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
 BackDaysCount=${BACKDAYS_COUNT:-30}
 
+for SourceHost in cloud28 cloud29; do
+    echo "============= ${SourceHost}"
+    Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/crypto/${SourceHost} ${FileGlob} ${BackDaysCount}"
+    echo $Cmd
 
-Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/crypto/${SourceHost} ${FileGlob} ${BackDaysCount}"
+    DATES=$($Cmd)
+
+    for dt in ${DATES}; do
+        echo $dt
+    done
+done
+
+DEFAULT_FGLOB=*.crypto_sim_md.db.gz
+FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
+
+echo "============= sim"
+Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/crypto/sim ${FileGlob} ${BackDaysCount}"
 echo $Cmd
 
 DATES=$($Cmd)
@@ -24,3 +33,4 @@ DATES=$($Cmd)
 for dt in ${DATES}; do
    echo $dt
 done
+

retrofit/crypto_sim_missing_dates.sh

@@ -1,22 +0,0 @@
-#!/bin/bash
-
-# DAILY_SIM_CRYPTO_DATE=20250725 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up daily_sim_crypto 
-#
-#
-source $(realpath $(dirname $0))/get_retrofit_dates.sh
-
-DEFAULT_FGLOB=*.crypto_sim_md.db.gz
-
-Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn}
-FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
-BackDaysCount=${BACKDAYS_COUNT:-30}
-
-Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/crypto/sim ${FileGlob} ${BackDaysCount}"
-echo $Cmd
-
-DATES=$($Cmd)
-
-for dt in ${DATES}; do
-   echo $dt
-done
-

retrofit/equity_missing_dates.sh

@@ -11,6 +11,7 @@ Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn}
 FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
 BackDaysCount=${BACKDAYS_COUNT:-50}
 
+echo "======================= Alpaca MD"
 Cmd="get_equity_retrofit_dates  ${Source} /works/cvtt/md_archive/equity/alpaca_md/2025/N/NVDA ${FileGlob} ${BackDaysCount}"
 echo $Cmd
 
@@ -19,3 +20,26 @@ DATES=$($Cmd)
 for dt in ${DATES}; do
    echo $dt
 done
+# ======================================
+
+DEFAULT_FGLOB=*.alpaca_sim_md.db.gz
+FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
+
+echo "======================= sim"
+Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/equity/alpaca_md/sim ${FileGlob} ${BackDaysCount}"
+echo $Cmd
+
+DATES=$($Cmd)
+
+for dt in ${DATES}; do
+    weekday=$(date -d "$dt" +%u)
+    (( weekday >= 6 )) && continue
+    is_equity_business_date "$dt"
+    rc=$?
+    if (( rc != 0 )); then
+        continue
+    fi
+     
+    fdt=$(date -d "$dt" +%Y-%m-%d 2>/dev/null)
+    echo $fdt
+done

retrofit/equity_sim_missing_dates.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-
-#
-# DAILY_SIM_EQUITY_DATE=20250723 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up daily_sim_equity 
-#
-source $(realpath $(dirname $0))/get_retrofit_dates.sh
-
-DEFAULT_FGLOB=*.alpaca_sim_md.db.gz
-
-Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn}
-FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
-BackDaysCount=${BACKDAYS_COUNT:-50}
-
-Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/equity/alpaca_md/sim ${FileGlob} ${BackDaysCount}"
-echo $Cmd
-
-DATES=$($Cmd)
-
-for dt in ${DATES}; do
-      weekday=$(date -d "$dt" +%u)
-      (( weekday >= 6 )) && continue
-
-      fdt=$(date -d "$dt" +%Y-%m-%d 2>/dev/null)
-      echo $fdt
-done

retrofit/get_retrofit_dates.sh

@@ -61,6 +61,30 @@ get_retrofit_dates() {
     printf '%s\n' "${missing_dates[@]}"
 }
 
+is_equity_business_date() {
+    local ymd=${1:-}
+
+    if [[ ! "$ymd" =~ ^[0-9]{8}$ ]]; then
+        echo "is_equity_business_date: expected YYYYMMDD, got \"$ymd\"" >&2
+        return 2
+    fi
+
+    local iso_date="${ymd:0:4}-${ymd:4:2}-${ymd:6:2}"
+    local url="https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS&start=${iso_date}&end=${iso_date}"
+    local response
+
+    if ! response=$(curl -fsS "$url" 2>/dev/null); then
+        echo "is_equity_business_date: failed to query trading calendar API" >&2
+        return 1
+    fi
+
+    if [[ "$response" =~ \[\s*\] ]]; then
+        return 1
+    fi
+
+    return 0
+}
+
 get_equity_retrofit_dates() {
     local host=${1:-}
     local root_dir=${2:-}
@@ -107,10 +131,15 @@ get_equity_retrofit_dates() {
     fi
 
     local -a missing_dates=()
-    local idx dt tmp
+    local idx dt tmp rc
     for (( idx = ${#target_dates[@]} - 1; idx >= 0; idx-- )); do
         dt=${target_dates[$idx]}
         [[ -n "${existing_dates[$dt]:-}" ]] && continue
+        is_equity_business_date "$dt"
+        rc=$?
+        if (( rc != 0 )); then
+            continue
+        fi
         if ! tmp=$(date -d "$dt" +%Y-%m-%d 2>/dev/null); then
             echo "invalid date \"$dt\"" >&2
             return 1