fix

docker/market_data/hist/alpaca_md_day.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-date_to_load=${1}
+date_to_load=${1} # YYYY-MM-DD
 
 DockerRegistry=cloud21.cvtt.vpn:5500
 DockerImage=${DockerRegistry}/alpaca_md_day #:latest
@@ -46,7 +46,7 @@ get_prev_business_day() {
 export -f get_prev_business_day
 
 if [ -z ${date_to_load} ] ; then
-    echo "Yesterday data has priority. Running historical container will be stopped"
+    echo "Date is not specified. Yesterday data has priority. Running historical container will be stopped"
     Cmd="docker stop ${ContainerName}"
     echo ${Cmd} && eval ${Cmd}
     Cmd="docker kill ${ContainerName}"
@@ -62,6 +62,7 @@ if [ -z ${date_to_load} ] ; then
         exit 3
     fi
 else
+    date_to_load=$(date -d ${date_to_load} '+%Y-%m-%d') #expected format
     echo "Historical run ${date_to_load}"
     if is_container_running "$ContainerName"; then
         echo "Container ${ContainerName} is already running."

docker/market_data/hist/load_alpaca_hbar.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+usage() {
+  echo "Usage: $0  -N <num_symbols> [-L <LogDir>] [-d <YYYYMMDD Date>]"
+  exit 1
+}
+
+is_container_running() {
+    local container_name=$1
+
+    if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
+        return 0  # true
+    else
+        return 1  # false
+    fi
+}
+
+export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+
+is_business_day() {
+    dt=${1}
+
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+get_prev_business_day() {
+    Start=${1}
+    while true; do
+        if is_business_day ${Start}; then
+            break
+        fi
+        echo "${Start} is not business day in US" >&2
+        Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
+    done
+    echo ${Start}
+}
+export -f get_prev_business_day
+
+# ----- Settings
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/alpaca_hbar #:latest
+ContainerName=alpaca_hbar
+LogDir=/home/cvtt/prod/logs
+# ----- Settings
+
+while getopts ":d:N:L:" opt; do
+  case ${opt} in
+    d )
+      date_to_load=$OPTARG
+      ;;
+    N )
+      NumSymbols=$OPTARG
+      ;;
+    L )
+      LogDir=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+if is_container_running "$ContainerName"; then
+    echo "Container ${ContainerName} is already running."
+    exit 3
+fi
+
+if [ -z "${date_to_load}" ]; then
+    echo "date_to_load is empty"
+    date_to_load=$(get_prev_business_day $(date  -d "yesterday" '+%Y-%m-%d'))
+    echo "Historical Data for ${date_to_load}"
+fi
+
+echo "date_to_load=${date_to_load}"
+
+Cmd="docker run"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ContainerName}"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} --volume=${LogDir}:/logs"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} -d ${date_to_load}"
+if [ -n "${NumSymbols}" ]; then
+    Cmd="${Cmd} -N ${NumSymbols}"
+fi
+
+echo $Cmd
+eval $Cmd
+
+if [ "$?" != "0" ] ; then
+    exit 1 # if killed we do not save last day
+fi

docker/market_data/hist/load_alpaca_qat.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+
+usage() {
+  echo "Usage: $0  -S <symbols> [-L <LogDir>] [-d <YYYYMMDD Date>]"
+  exit 1
+}
+
+is_container_running() {
+    local container_name=$1
+
+    if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
+        return 0  # true
+    else
+        return 1  # false
+    fi
+}
+
+export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+
+is_business_day() {
+    dt=${1}
+
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+get_prev_business_day() {
+    Start=${1}
+    while true; do
+        if is_business_day ${Start}; then
+            break
+        fi
+        echo "${Start} is not business day in US" >&2
+        Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
+    done
+    echo ${Start}
+}
+export -f get_prev_business_day
+
+# ----- Settings
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/alpaca_qat #:latest
+ContainerName=alpaca_qat
+LogDir=/home/cvtt/prod/logs
+# ----- Settings
+
+while getopts ":d:S:L:" opt; do
+  case ${opt} in
+    d )
+      date_to_load=$OPTARG
+      ;;
+    S )
+      Symbols=$OPTARG
+      ;;
+    L )
+      LogDir=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+if [ -z ${Symbols}  ] ; then
+    echo "symbols are not specified"
+    usage
+fi
+
+if is_container_running "$ContainerName"; then
+    echo "Container ${ContainerName} is already running."
+    exit 3
+fi
+
+if [ -z "${date_to_load}" ]; then
+    echo "date_to_load is empty"
+    date_to_load=$(get_prev_business_day $(date  -d "yesterday" '+%Y-%m-%d'))
+    echo "Historical Data for ${date_to_load}"
+fi
+
+echo "date_to_load=${date_to_load}"
+
+Cmd="docker run"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ContainerName}"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} --volume=${LogDir}:/logs"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} -d ${date_to_load}"
+Cmd="${Cmd} -S ${Symbols}"
+
+echo $Cmd
+eval $Cmd
+
+if [ "$?" != "0" ] ; then
+    exit 1 # if killed we do not save last day
+fi

release_version.txt

@@ -1 +1 @@
-1.6.8,cleaned out several docker scripts. added checklist sources
+1.7.5.fx1,using new(rust) alpaca market data structure for hbar

research/load_eqty_md.sh

@@ -167,10 +167,10 @@ EOF
 echo "Loading files"
 for stock in "${Stocks[@]}"; do
     StockLetter="${stock:0:1}"
-    SourceFile=$(${date} -d ${md_date} "+%Y%m%d.${stock}.1min.db")
+    SourceFile=$(${date} -d ${md_date} "+%Y%m%d.${stock}.alpaca_1m_bars.db")
 
     SourceFileZip="${SourceFile}.gz"
-    SourceFilePath=$(${date} -d ${md_date} "+${SourceRootDir}/${StockLetter}/${stock}/%Y/${SourceFileZip}")
+    SourceFilePath=$(${date} -d ${md_date} "+${SourceRootDir}/%Y/${StockLetter}/${stock}/${SourceFileZip}")
     echo ${SourceFilePath}
 
     download_file ${SourceFilePath}
@@ -180,6 +180,3 @@ if [ -f ${TargetFilePath} ]; then
     chmod 600
     ls -lh ${TargetFilePath}
 fi
-
-# mkdir -p /tmp/aaa
-# cp -r ${tmp_dir}/* /tmp/aaa/

scripts/healthcheck/storage_health_check.sh

@@ -69,7 +69,9 @@ function storage_check() {
         Cmd="ssh -p ${port} $host"
         Cmd="${Cmd} eval \"df -hTl"
         Cmd="${Cmd} -x squashfs"
-        Cmd="${Cmd} | grep -v tmpfs"
+        Cmd="${Cmd} -x tmpfs"
+        Cmd="${Cmd} -x vfat"
+        Cmd="${Cmd} -x devtmpfs"
         Cmd="${Cmd} | grep -v Filesystem\""
 
         IFS=$'\n' ; lines=$(eval ${Cmd})