improvements to alpaca md

release_version.txt

@@ -1 +1 @@
-0.6.4
+0.6.8

scripts/alpaca_hist_md/alpaca_assets.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+CredsFile=${1}
+CredKey=${2}
+
+if [ "${CredsFile}" == "" ] ; then
+    CredsFile="${HOME}/.creds"
+fi
+
+if [ "${CredKey}" == "" ] ; then
+    CredKey="ALPACA_SANDBOX"
+fi
+
+key=$(jq -r ".[\"${CredKey}\"] | .api_key" ${CredsFile})
+secret=$(jq -r ".[\"${CredKey}\"] | .secret_key" ${CredsFile})
+
+Cmd="curl -s --request GET --url 'https://paper-api.alpaca.markets/v2/assets?status=active'"
+Cmd="${Cmd} --header 'APCA-API-KEY-ID: ${key}'"
+Cmd="${Cmd} --header 'APCA-API-SECRET-KEY: ${secret}'"
+Cmd="${Cmd} --header 'accept: application/json'"
+
+eval ${Cmd}
+
+# split string into array
+Instruments=()
+for Inst in $(eval ${Cmd})
+do
+    Instruments+=("$Inst")
+done

scripts/alpaca_hist_md/alpaca_md.sh

@@ -1,6 +1,11 @@
 #!/bin/bash
 
-CalendarURL=http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+Start=${1}
+NumJobs=${2}
+InstListFile=${3}
+
+export CalendarURL=http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+
 is_business_day() {
     dt=${1}
 
@@ -14,13 +19,9 @@ is_business_day() {
 }
 export -f is_business_day
 
-Start=${1}
-End=${2}
-NumJobs=${3}
 
 if [ "${Start}" == "" ] ; then
-    echo "Usage: ${0} <start_date> [<end_date>] [<num_jobs> (25)]]"
+    Start=$(date -d "yesterday" "+%Y-%m-%d")
-    exit
 fi
 
 if ! is_business_day ${Start}; then
@@ -28,23 +29,23 @@ if ! is_business_day ${Start}; then
     exit
 fi
 
-if [ "${End}" == "" ] ; then
 End=$(date -d "${Start} + 1 day" "+%Y-%m-%d")
-fi
 
 if [ "${NumJobs}" == "" ] ; then
-    NumJobs=25
+    NumJobs=30
 fi
 
 echo "Start=${Start} End=${End} NumJobs=${NumJobs}"
 
+export PYTHONPATH=/home/cvtt/prod
+export Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3
+export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_mkt_data.py
 
-Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3
+export OutputDir=/home/cvtt/host_drive/alpaca_md # Local
-PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_mkt_data.py
+export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
-
+export LogDir=/home/cvtt/prod/logs/alpaca_md
-OutputDir=/home/cvtt/alpaca_md
-Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
 
+mkdir -p ${LogDir}
 
 run_proc() {
     Inst=${1}
@@ -52,7 +53,6 @@ run_proc() {
     End=${3}
     echo "Running for $Inst"
 
-
     Cmd="${Python} ${PyScript}"
     Cmd="${Cmd} --config=${Config}"
     Cmd="${Cmd} --output_dir=${OutputDir}"
@@ -61,14 +61,15 @@ run_proc() {
     if [ "${End}" != "" ]; then
         Cmd="${Cmd} --end=${End}"
     fi
-    Cmd="${Cmd} --log_file=./logs/${Start}.${Inst}.log"
+    Cmd="${Cmd} --log_file=${LogDir}/${Start}.${Inst}.log"
     echo ${Cmd}
     eval ${Cmd}
 }
 
-export PYTHONPATH=/home/cvtt/prod
 export -f run_proc
 
+if [ -n $InstListFile ]; then
+
     key=$(jq -r '.["ALPACA_SANDBOX"] | .api_key' ~/.creds)
     secret=$(jq -r '.["ALPACA_SANDBOX"] | .secret_key' ~/.creds)
 
@@ -79,6 +80,9 @@ Cmd="${Cmd} --header 'accept: application/json'"
     Cmd="${Cmd} | jq '.[] | select(.class == \"us_equity\" and .exchange != \"OTC\") | .symbol'"
     Cmd="${Cmd}  | sed 's/\"//g'"
     Cmd="${Cmd} | sed 's/^/STOCK-/'"
+else
+    Cmd="cat ${InstListFile}"
+fi
 
 # split string into array
 Instruments=()
@@ -87,7 +91,7 @@ do
     Instruments+=("$Inst")
 done
 
-slice_size=100 # 10K symbols parallel cannot handle
+slice_size=500 # 10K symbols parallel cannot handle
 for ((ii=0; ii <${#Instruments[@]}; ii+=slice_size)); do
     InstSlice=("${Instruments[@]:ii:slice_size}")
 

scripts/alpaca_hist_md/run_alpaca_hist_md.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+is_business_day() {
+    dt=${1}
+
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+# Dates are in descending order
+FinishedDateFile=${HOME}/prod/run/alpaca_md_last_date
+Script=${HOME}/prod/run/alpaca_md.sh
+
+last_date=$(cat ${FinishedDateFile})
+
+if [ "${last_date}" == "" ]; then
+    echo "process is currently on"
+    exit
+fi
+
+while true; do
+    new_date=$(date -d "${last_date} - 1 day" "+%Y-%m-%d")
+    if is_business_day ${new_date}; then
+        break
+    else
+        last_date=${new_date}
+    fi
+done
+
+
+echo Collecting data for ${new_date}
+
+Cmd="${Script} ${new_date}"
+
+echo "${0} Starting at $(date)"
+
+echo > ${FinishedDateFile}
+echo ${Cmd} && eval ${Cmd}
+echo $new_date > ${FinishedDateFile}
+
+echo "${0} Done at $(date)"
+
+