progress

healthcheck/hosts_check.sh

@@ -9,10 +9,9 @@
 #         "timeout_sec": 5
 #     },
 
-RootDir="${HOME}/prod"
 
 AlertChannel=Alerts-CVTT
-Sender=${RootDir}/ops/utils/send_mmost.sh
+Sender=/works/utils/send_mmost.sh
 ConfigUrl=http://cloud16.cvtt.vpn:6789/admin/cvtt_hosts
 
 HOSTS_CONFIG=$(curl -s ${ConfigUrl} | ${HOME}/bin/hjson -j)

healthcheck/md/crypto_missing_dates.sh

@@ -6,12 +6,13 @@ source ${function_file}
 DEFAULT_FGLOB=*.mktdata.db.gz
 
 Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn}
+SrcRoot=${MD_SRC_ROOT}/works/cvtt/md_archive/crypto
 FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
 BackDaysCount=${BACKDAYS_COUNT:-30}
 
 for SourceHost in cloud28 cloud29; do
     echo "============= ${SourceHost}"
-    Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/crypto/${SourceHost} ${FileGlob} ${BackDaysCount}"
+    Cmd="get_retrofit_dates  ${Source} ${SrcRoot}/${SourceHost} ${FileGlob} ${BackDaysCount}"
     echo $Cmd
 
     DATES=$($Cmd)
@@ -25,7 +26,7 @@ DEFAULT_FGLOB=*.crypto_sim_md.db.gz
 FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
 
 echo "============= sim"
-Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/crypto/sim ${FileGlob} ${BackDaysCount}"
+Cmd="get_retrofit_dates  ${Source} ${SrcRoot}/sim ${FileGlob} ${BackDaysCount}"
 echo $Cmd
 
 DATES=$($Cmd)

healthcheck/md/equity_missing_dates.sh

@@ -8,11 +8,12 @@ source ${function_file}
 DEFAULT_FGLOB=*.alpaca_1m_bars.db.gz
 
 Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn}
+SrcRoot="${MD_SRC_ROOT}/works/cvtt/md_archive/equity"
 FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
 BackDaysCount=${BACKDAYS_COUNT:-50}
 
 echo "======================= Alpaca MD"
-Cmd="get_equity_retrofit_dates  ${Source} /works/cvtt/md_archive/equity/alpaca_md/2025/N/NVDA ${FileGlob} ${BackDaysCount}"
+Cmd="get_equity_retrofit_dates  ${Source} ${SrcRoot}/alpaca_md/*/N/NVDA/${FileGlob} ${BackDaysCount}"
 echo $Cmd
 
 DATES=$($Cmd)
@@ -26,7 +27,7 @@ DEFAULT_FGLOB=*.alpaca_sim_md.db.gz
 FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
 
 echo "======================= sim"
-Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/equity/alpaca_md/sim ${FileGlob} ${BackDaysCount}"
+Cmd="get_retrofit_dates  ${Source} ${SrcRoot}/alpaca_md/sim ${FileGlob} ${BackDaysCount}"
 echo $Cmd
 
 DATES=$($Cmd)

healthcheck/md/get_retrofit_dates.sh

@@ -86,12 +86,12 @@ is_equity_business_date() {
 }
 
 get_equity_retrofit_dates() {
+    # get_equity_retrofit_dates cvtt@cloud21.cvtt.vpn /works/cvtt/md_archive/equity/alpaca_md/*/N/NVDA/*.alpaca_1m_bars.db.gz 50
     local host=${1:-}
-    local root_dir=${2:-}
+    local filename_glob=${2:-}
-    local filename_glob=${3:-}
+    local day_count=${3:-}
-    local day_count=${4:-}
 
-    if [[ -z "$host" || -z "$root_dir" || -z "$filename_glob" || -z "$day_count" ]]; then
+    if [[ -z "$host" || -z "$filename_glob" || -z "$day_count" ]]; then
         echo "usage: get_equity_retrofit_dates <host> <root_dir> <filename_glob> <day_count>" >&2
         return 1
     fi
@@ -118,13 +118,14 @@ get_equity_retrofit_dates() {
     declare -A existing_dates=()
 
     local remote_cmd remote_output entry
-    printf -v remote_cmd "cd %q 2>/dev/null && LC_ALL=C ls -1 %s" "$root_dir" "$filename_glob"
+    remote_cmd="ls ${filename_glob}"
     remote_output=$(ssh "$host" "$remote_cmd" 2>/dev/null || true)
 
     if [[ -n "$remote_output" ]]; then
         while IFS= read -r entry; do
             [[ -z "$entry" ]] && continue
-            if [[ "$entry" =~ ^([0-9]{8}) ]]; then
+            fname=$(basename $entry)
+            if [[ "$fname" =~ ^([0-9]{8}) ]]; then
                 existing_dates["${BASH_REMATCH[1]}"]=1
             fi
         done <<<"$remote_output"

healthcheck/md/md_checklist.sh

@@ -8,17 +8,14 @@ usage() {
 
 echo Starting $0 ...
 
-RootDir="${HOME}/prod"
-# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
-
-export PYTHONPATH=${RootDir}
-
 StatusChannel="MD-Status"
-Sender=${RootDir}/ops/utils/send_mmost.sh
+Sender=/works/utils/send_mmost.sh
 
 # ----- For DEBUGGING
 #Sender=cat
 #StatusChannel=
+# ----- For DEBUGGING
+
 SDir=$(realpath $(dirname $0))
 
 run_crypto_checklist() {
@@ -32,7 +29,7 @@ run_crypto_checklist() {
     declare -A Commands
     Commands=(
         ["crypto:missing-dates:cloud21"]="MD_HOST=cloud21.cvtt.vpn ${SDir}/crypto_missing_dates.sh"
-        ["crypto:missing-dates:hs01"]="MD_HOST=hs01.cvtt.vpn ${SDir}/crypto_missing_dates.sh"
+        ["crypto:missing-dates:hl-md-archiver"]="MD_HOST=hl-md-archiver.cvtt.vpn MD_SRC_ROOT=/mnt/nas-01 ${SDir}/crypto_missing_dates.sh"
     )
 
     for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
@@ -58,7 +55,7 @@ run_eqt_checklist() {
     declare -A Commands
     Commands=(
         ["equity:missing-dates:cloud21"]="MD_HOST=cloud21.cvtt.vpn ${SDir}/equity_missing_dates.sh"
-        ["equity:missing-dates:hs01"]="MD_HOST=hs01.cvtt.vpn ${SDir}/equity_missing_dates.sh"
+        ["equity:missing-dates:hl-md-archiver"]="MD_HOST=hl-md-archiver.cvtt.vpn MD_SRC_ROOT=/mnt/nas-01 ${SDir}/equity_missing_dates.sh"
     )    
 
     for name in $(printf "%s\n" "${!Commands[@]}" | sort); do

healthcheck/services_check.sh

@@ -7,11 +7,8 @@
     #     "port": 8080
     # },
 
-RootDir="${HOME}/prod"
-# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
-
 AlertChannel=Alerts-CVTT
-Sender=${RootDir}/ops/utils/send_mmost.sh
+Sender=/works/utils/send_mmost.sh
 ConfigUrl=http://cloud16.cvtt.vpn:6789/admin/cvtt_services
 
 SERVICES_CONFIG=$(curl -s ${ConfigUrl} | ${HOME}/bin/hjson -j)

healthcheck/storage_health_check.sh

@@ -1,14 +1,9 @@
 #!/bin/bash
 echo $0 $* | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
 
-RootDir="${HOME}/prod"
-
-export PYTHONPATH=${RootDir}
-
-
 StatusChannel=Status-CVTT
 AlertChannel=Alerts-CVTT
-Sender=${RootDir}/ops/utils/send_mmost.sh
+Sender=/works/utils/send_mmost.sh
 
 # ----- For DEBUGGING
 # Sender=cat