cleaning

2025-05-17 22:20:51 -04:00
parent e083eb27d9
commit ccf55b8e66
4 changed files with 0 additions and 0 deletions
@@ -0,0 +1,34 @@
+FROM linuxcontainers/debian-slim:latest
+
+ARG FROM_DIR=./docker_dev/market_data/alpaca_hbar
+
+# Update the package list and install required packages
+RUN apt-get update && apt-get install -y \
+    --no-install-recommends \
+    apt-utils \
+    rsync \
+    openssh-client \
+    software-properties-common \
+    curl \
+    jq
+
+# must be from disted version
+COPY cvtt-rust/bin/alpaca_md_hbar_loader /
+
+# COPY ${FROM_DIR}/.creds  /.creds
+COPY ${FROM_DIR}/alpaca_hbar.sh /alpaca_hbar.sh
+
+COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
+COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
+
+RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub
+
+
+# Shared Volumes
+RUN mkdir -p /data
+RUN mkdir /logs
+RUN chmod +x /alpaca_hbar.sh
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENTRYPOINT [ "/alpaca_hbar.sh" ]
@@ -0,0 +1,108 @@
+#!/bin/bash
+
+usage() {
+  echo "Usage: $0 [-d <YYYYMMDD Date>] [-N <num_symbols>]"
+  exit 1
+}
+
+is_business_day() {
+    dt=${1}
+
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+# --- Settings
+export Bin=/alpaca_md_hbar_loader
+export OutputDir=/data/alpaca_md # Local
+export LogDir=/logs
+# --- Settings
+
+while getopts ":d:N:" opt; do
+  case ${opt} in
+    d )
+      Start=$OPTARG
+      ;;
+    N )
+      NumSymbols=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+
+mkdir -p ${OutputDir}
+mkdir -p ${LogDir}
+
+
+echo "CommandLine: ${*}"
+
+export CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS
+
+
+if [ -z ${Start}  ] ; then
+    echo "start is not specified, getting yesterday..."
+    Start=$(date -d "yesterday" "+%Y-%m-%d")
+    echo "Start=${Start}"
+fi
+
+while true; do
+    if is_business_day ${Start}; then
+        break
+    fi
+    echo "${Start} is not business day in US"
+    Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
+done
+
+
+echo "Start=${Start}"
+Dstamp=$(date -d "${Start}" "+%Y%m%d")
+
+Cmd="${Bin}"
+Cmd="${Cmd} --credentials_file=/.creds"
+Cmd="${Cmd} --output_directory=${OutputDir}"
+Cmd="${Cmd} --start=${Start}"
+if [ -n "${NumSymbols}" ]; then
+    Cmd="${Cmd} --num_symbols=${NumSymbols}"
+fi
+Cmd="${Cmd}  > ${LogDir}/${Dstamp}.alpaca_hbar.log 2>&1"
+
+echo ${Cmd}
+eval ${Cmd}
+
+echo "Compressing"
+for file in $(find ${OutputDir} -type f -name '*db' -print )
+do
+    echo "Compressing ${file}"
+    gzip ${file}
+done
+
+
+Source=/data/
+Targets=
+Targets="${Targets} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity"
+Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
+
+
+for tgt in ${Targets}
+do
+    Cmd="/usr/bin/rsync -ahv"
+    Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
+    Cmd+=" ${Source} ${tgt}"
+    echo $Cmd
+    eval $Cmd
+done
+