From c2dae20d7bfd25858abbbe64b1d3ccb7e1b7a751 Mon Sep 17 00:00:00 2001
From: Oleg Sheynin <osheynin@gmail.com>
Date: Tue, 26 Mar 2024 11:30:41 -0400
Subject: [PATCH] improvements to alpaca md

---
 release_version.txt                          |  2 +-
 scripts/alpaca_hist_md/alpaca_md.sh          | 16 +++----
 scripts/alpaca_hist_md/run_alpaca_hist_md.sh | 49 ++++++++++++++++++++
 3 files changed, 56 insertions(+), 11 deletions(-)
 create mode 100755 scripts/alpaca_hist_md/run_alpaca_hist_md.sh

diff --git a/release_version.txt b/release_version.txt
index 8b707c6..bc8443e 100644
--- a/release_version.txt
+++ b/release_version.txt
@@ -1 +1 @@
-0.6.7
\ No newline at end of file
+0.6.8
\ No newline at end of file
diff --git a/scripts/alpaca_hist_md/alpaca_md.sh b/scripts/alpaca_hist_md/alpaca_md.sh
index 6e3b82c..08c9b2e 100755
--- a/scripts/alpaca_hist_md/alpaca_md.sh
+++ b/scripts/alpaca_hist_md/alpaca_md.sh
@@ -1,9 +1,8 @@
 #!/bin/bash
 
 Start=${1}
-End=${2}
-NumJobs=${3}
-InstListFile=${4}
+NumJobs=${2}
+InstListFile=${3}
 
 export CalendarURL=http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
 
@@ -30,12 +29,10 @@ if ! is_business_day ${Start}; then
     exit
 fi
 
-if [ "${End}" == "" ] ; then
-    End=$(date -d "${Start} + 1 day" "+%Y-%m-%d")
-fi
+End=$(date -d "${Start} + 1 day" "+%Y-%m-%d")
 
 if [ "${NumJobs}" == "" ] ; then
-    NumJobs=25
+    NumJobs=30
 fi
 
 echo "Start=${Start} End=${End} NumJobs=${NumJobs}"
@@ -44,8 +41,7 @@ export PYTHONPATH=/home/cvtt/prod
 export Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3
 export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_mkt_data.py
 
-# export OutputDir=/home/cvtt/alpaca_md
-export OutputDir=/home/cvtt/host_drive/alpaca_md
+export OutputDir=/home/cvtt/host_drive/alpaca_md # Local
 export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
 export LogDir=/home/cvtt/prod/logs/alpaca_md
 
@@ -95,7 +91,7 @@ do
     Instruments+=("$Inst")
 done
 
-slice_size=100 # 10K symbols parallel cannot handle
+slice_size=500 # 10K symbols parallel cannot handle
 for ((ii=0; ii <${#Instruments[@]}; ii+=slice_size)); do
     InstSlice=("${Instruments[@]:ii:slice_size}")
 
diff --git a/scripts/alpaca_hist_md/run_alpaca_hist_md.sh b/scripts/alpaca_hist_md/run_alpaca_hist_md.sh
new file mode 100755
index 0000000..f194230
--- /dev/null
+++ b/scripts/alpaca_hist_md/run_alpaca_hist_md.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+
+is_business_day() {
+    dt=${1}
+
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+# Dates are in descending order
+FinishedDateFile=${HOME}/prod/run/alpaca_md_last_date
+Script=${HOME}/prod/run/alpaca_md.sh
+
+last_date=$(cat ${FinishedDateFile})
+
+if [ "${last_date}" == "" ]; then
+    echo "process is currently on"
+    exit
+fi
+
+while true; do
+    new_date=$(date -d "${last_date} - 1 day" "+%Y-%m-%d")
+    if is_business_day ${new_date}; then
+        break
+    else
+        last_date=${new_date}
+    fi
+done
+
+
+echo Collecting data for ${new_date}
+
+Cmd="${Script} ${new_date}"
+
+echo "${0} Starting at $(date)"
+
+echo > ${FinishedDateFile}
+echo ${Cmd} && eval ${Cmd}
+echo $new_date > ${FinishedDateFile}
+
+echo "${0} Done at $(date)"
+
+