progress with retrofit

.env

@@ -1,7 +1,7 @@
 CREDS_FILE=/config/.creds
 
 # ==================================== daily_crypto_md
-CRYPTO_IMAGE_VERSION=0.0.4
+CRYPTO_IMAGE_VERSION=0.0.6
 CRYPTO_CONFIG_FILE=http://cloud16.cvtt.vpn:6789/apps/utils/mktdata/daily_md_crypto
 CRYPTO_CONTAINER_NAME_PFX=daily_crypto_md
 CRYPTO_DB_SCHEMAS=coinbase,bnbspot
@@ -9,7 +9,7 @@ CRYPTO_DB_SCHEMAS=coinbase,bnbspot
 CRYPTO_MD_DATE=
 
 CRYPTO_RSYNC_TARGETS=
-CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto"
+CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/works/cvtt/md_archive/crypto"
 CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto"
 
 CRYPTO_ADD_ARGS="--log_stdout"
@@ -20,7 +20,7 @@ CRYPTO_PYTHON_DEBUG_WAIT=Y
 CRYPTO_PYTHON_DEBUG_PORT=45678
 
 # ==================================== daily_equity_md
-EQUITY_IMAGE_VERSION=0.0.2
+EQUITY_IMAGE_VERSION=0.0.7
 EQUITY_CONTAINER_NAME_PFX=daily_equity_md
 EQUITY_HBAR_NUM_SYMBOLS=
 EQUITY_QAT_SYMBOLS=COIN,GBTC,HOOD,MSTR,PYPL,QQQ,NVDA
@@ -28,22 +28,19 @@ EQUITY_QAT_SYMBOLS=COIN,GBTC,HOOD,MSTR,PYPL,QQQ,NVDA
 EQUITY_HBAR_DATE=
 
 EQUITY_RSYNC_TARGETS=
-EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity"
+EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md"
-EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
+EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md"
-# EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/test"
 
 # ==================================== daily_sim_crypto
 SIM_CRYPTO_IMAGE_VERSION=0.0.3
 DAILY_SIM_CRYPTO_DATE=
 
-#DAILY_SIM_CRYPTO_SOURCE_HOST=cloud21.cvtt.vpn
-#DAILY_SIM_CRYPTO_SOURCE_DIR=/opt/store/cvtt/md_archive/crypto
 DAILY_SIM_CRYPTO_SOURCE_HOST=hs01.cvtt.vpn
 DAILY_SIM_CRYPTO_SOURCE_DIR=/works/cvtt/md_archive/crypto
 DAILY_SIM_CRYPTO_DB_SOURCES=cloud28,cloud29
 DAILY_SIM_CRYPTO_RSYNC_TARGETS=
 DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/sim"
-DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim"
+DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/works/cvtt/md_archive/crypto/sim"
 
 
 # ==================================== daily_sim_equity
@@ -51,10 +48,10 @@ SIM_EQUITY_IMAGE_VERSION=0.0.2
 DAILY_SIM_EQUITY_DATE=
 
 DAILY_SIM_EQUITY_SOURCE_HOST=cloud21.cvtt.vpn
-DAILY_SIM_EQUITY_SOURCE_DIR=/opt/store/cvtt/md_archive/equity/alpaca_md
+DAILY_SIM_EQUITY_SOURCE_DIR=/works/cvtt/md_archive/equity/alpaca_md
 DAILY_SIM_EQUITY_SYMBOL_LIST=COIN #,CRCL,GBTC,HOOD,MSTR
 DAILY_SIM_EQUITY_RSYNC_TARGETS=
 DAILY_SIM_EQUITY_RSYNC_TARGETS="${DAILY_SIM_EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim"
-DAILY_SIM_EQUITY_RSYNC_TARGETS="${DAILY_SIM_EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim"
+DAILY_SIM_EQUITY_RSYNC_TARGETS="${DAILY_SIM_EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim"
 
 

.gitignore

@@ -3,3 +3,4 @@ logs/
 nohup.out
 .history/
 __OLD__/
+__DELETE__/

retrofit/crypto_missing_dates.sh

@@ -1,11 +1,5 @@
 #!/bin/bash
 
-SourceHost=${1}
-if [ "${SourceHost}" == "" ]; then
-   echo "Usage: ${0} <source_host>"
-   exit 1
-fi
-
 function_file=$(realpath $(dirname $0))/get_retrofit_dates.sh
 source ${function_file}
 
@@ -15,8 +9,23 @@ Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn}
 FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
 BackDaysCount=${BACKDAYS_COUNT:-30}
 
+for SourceHost in cloud28 cloud29; do
+    echo "============= ${SourceHost}"
+    Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/crypto/${SourceHost} ${FileGlob} ${BackDaysCount}"
+    echo $Cmd
 
-Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/crypto/${SourceHost} ${FileGlob} ${BackDaysCount}"
+    DATES=$($Cmd)
+
+    for dt in ${DATES}; do
+        echo $dt
+    done
+done
+
+DEFAULT_FGLOB=*.crypto_sim_md.db.gz
+FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
+
+echo "============= sim"
+Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/crypto/sim ${FileGlob} ${BackDaysCount}"
 echo $Cmd
 
 DATES=$($Cmd)
@@ -24,3 +33,4 @@ DATES=$($Cmd)
 for dt in ${DATES}; do
    echo $dt
 done
+

retrofit/crypto_sim_missing_dates.sh

@@ -1,22 +0,0 @@
-#!/bin/bash
-
-# DAILY_SIM_CRYPTO_DATE=20250725 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up daily_sim_crypto 
-#
-#
-source $(realpath $(dirname $0))/get_retrofit_dates.sh
-
-DEFAULT_FGLOB=*.crypto_sim_md.db.gz
-
-Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn}
-FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
-BackDaysCount=${BACKDAYS_COUNT:-30}
-
-Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/crypto/sim ${FileGlob} ${BackDaysCount}"
-echo $Cmd
-
-DATES=$($Cmd)
-
-for dt in ${DATES}; do
-   echo $dt
-done
-

retrofit/equity_missing_dates.sh

@@ -11,6 +11,7 @@ Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn}
 FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
 BackDaysCount=${BACKDAYS_COUNT:-50}
 
+echo "======================= Alpaca MD"
 Cmd="get_equity_retrofit_dates  ${Source} /works/cvtt/md_archive/equity/alpaca_md/2025/N/NVDA ${FileGlob} ${BackDaysCount}"
 echo $Cmd
 
@@ -19,3 +20,26 @@ DATES=$($Cmd)
 for dt in ${DATES}; do
    echo $dt
 done
+# ======================================
+
+DEFAULT_FGLOB=*.alpaca_sim_md.db.gz
+FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
+
+echo "======================= sim"
+Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/equity/alpaca_md/sim ${FileGlob} ${BackDaysCount}"
+echo $Cmd
+
+DATES=$($Cmd)
+
+for dt in ${DATES}; do
+    weekday=$(date -d "$dt" +%u)
+    (( weekday >= 6 )) && continue
+    is_equity_business_date "$dt"
+    rc=$?
+    if (( rc != 0 )); then
+        continue
+    fi
+     
+    fdt=$(date -d "$dt" +%Y-%m-%d 2>/dev/null)
+    echo $fdt
+done

retrofit/equity_sim_missing_dates.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-
-#
-# DAILY_SIM_EQUITY_DATE=20250723 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up daily_sim_equity 
-#
-source $(realpath $(dirname $0))/get_retrofit_dates.sh
-
-DEFAULT_FGLOB=*.alpaca_sim_md.db.gz
-
-Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn}
-FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
-BackDaysCount=${BACKDAYS_COUNT:-50}
-
-Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/equity/alpaca_md/sim ${FileGlob} ${BackDaysCount}"
-echo $Cmd
-
-DATES=$($Cmd)
-
-for dt in ${DATES}; do
-      weekday=$(date -d "$dt" +%u)
-      (( weekday >= 6 )) && continue
-
-      fdt=$(date -d "$dt" +%Y-%m-%d 2>/dev/null)
-      echo $fdt
-done

retrofit/get_retrofit_dates.sh

@@ -61,6 +61,30 @@ get_retrofit_dates() {
     printf '%s\n' "${missing_dates[@]}"
 }
 
+is_equity_business_date() {
+    local ymd=${1:-}
+
+    if [[ ! "$ymd" =~ ^[0-9]{8}$ ]]; then
+        echo "is_equity_business_date: expected YYYYMMDD, got \"$ymd\"" >&2
+        return 2
+    fi
+
+    local iso_date="${ymd:0:4}-${ymd:4:2}-${ymd:6:2}"
+    local url="https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS&start=${iso_date}&end=${iso_date}"
+    local response
+
+    if ! response=$(curl -fsS "$url" 2>/dev/null); then
+        echo "is_equity_business_date: failed to query trading calendar API" >&2
+        return 1
+    fi
+
+    if [[ "$response" =~ \[\s*\] ]]; then
+        return 1
+    fi
+
+    return 0
+}
+
 get_equity_retrofit_dates() {
     local host=${1:-}
     local root_dir=${2:-}
@@ -107,10 +131,15 @@ get_equity_retrofit_dates() {
     fi
 
     local -a missing_dates=()
-    local idx dt tmp
+    local idx dt tmp rc
     for (( idx = ${#target_dates[@]} - 1; idx >= 0; idx-- )); do
         dt=${target_dates[$idx]}
         [[ -n "${existing_dates[$dt]:-}" ]] && continue
+        is_equity_business_date "$dt"
+        rc=$?
+        if (( rc != 0 )); then
+            continue
+        fi
         if ! tmp=$(date -d "$dt" +%Y-%m-%d 2>/dev/null); then
             echo "invalid date \"$dt\"" >&2
             return 1