diff --git a/market_data/alpaca_hbar/.creds b/market_data/alpaca_hbar/.creds new file mode 100644 index 0000000..0a3b012 --- /dev/null +++ b/market_data/alpaca_hbar/.creds @@ -0,0 +1,7 @@ +{ + "__dummy__": null + , "ALPACA_SANDBOX": { + "api_key": "PKLZSLFZMFMN1R28K9HK" + , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA" + } +} diff --git a/market_data/alpaca_hbar/Dockerfile b/market_data/alpaca_hbar/Dockerfile new file mode 100644 index 0000000..4662a68 --- /dev/null +++ b/market_data/alpaca_hbar/Dockerfile @@ -0,0 +1,34 @@ +FROM linuxcontainers/debian-slim:latest + +ARG FROM_DIR=docker_dev/market_data/alpaca_hbar + +# Update the package list and install required packages +RUN apt-get update && apt-get install -y \ + --no-install-recommends \ + apt-utils \ + rsync \ + openssh-client \ + software-properties-common \ + curl \ + jq + +COPY cvtt-rust/target/release/alpaca_md_hbar_loader / + +COPY ${FROM_DIR}/.creds /.creds +COPY ${FROM_DIR}/alpaca_hbar.sh /alpaca_hbar.sh + +COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa +COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub +COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts + +RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts + + +# Shared Volumes +RUN mkdir -p /data +RUN mkdir /logs +RUN chmod +x /alpaca_hbar.sh + +WORKDIR / +SHELL ["/bin/bash", "-c"] +ENTRYPOINT [ "/alpaca_hbar.sh" ] diff --git a/market_data/alpaca_hbar/alpaca_hbar.sh b/market_data/alpaca_hbar/alpaca_hbar.sh new file mode 100755 index 0000000..2009283 --- /dev/null +++ b/market_data/alpaca_hbar/alpaca_hbar.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +usage() { + echo "Usage: $0 [-d ] [-N ]" + exit 1 +} + +is_business_day() { + dt=${1} + + open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time') + if [ -n "${open_time}" ]; then + return 0 + else + return 1 + fi + +} +export -f is_business_day + +# --- Settings +export Bin=/alpaca_md_hbar_loader +export OutputDir=/data/alpaca_md # Local +export LogDir=/logs +# --- Settings + +while getopts ":d:N:" opt; do + case ${opt} in + d ) + Start=$OPTARG + ;; + N ) + NumSymbols=$OPTARG + ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + usage + ;; + : ) + echo "Option -$OPTARG requires an argument." >&2 + usage + ;; + esac +done + +# if [ -z ${Symbols} ] ; then +# echo "symbols are not specified" +# usage +# fi + +mkdir -p ${OutputDir} +mkdir -p ${LogDir} + + +echo "CommandLine: ${*}" + +export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS + + +if [ -z ${Start} ] ; then + echo "start is not specified, getting yesterday..." + Start=$(date -d "yesterday" "+%Y-%m-%d") + echo "Start=${Start}" +fi + +while true; do + if is_business_day ${Start}; then + break + fi + echo "${Start} is not business day in US" + Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d") +done + + +echo "Start=${Start}" +Dstamp=$(date -d "${Start}" "+%Y%m%d") + +Cmd="${Bin}" +Cmd="${Cmd} --credentials_file=/.creds" +Cmd="${Cmd} --output_directory=${OutputDir}" +Cmd="${Cmd} --start=${Start}" +if [ -n "${NumSymbols}" ]; then + Cmd="${Cmd} --num_symbols=${NumSymbols}" +fi +Cmd="${Cmd} > ${LogDir}/${Dstamp}.alpaca_hbar.log 2>&1" + +echo ${Cmd} +eval ${Cmd} + +echo "Compressing" +for file in $(find ${OutputDir} -type f -name '*db' -print ) +do + echo "Compressing ${file}" + gzip ${file} +done + + +Source=/data/ +Targets= +Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity" +Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity" + + +for tgt in ${Targets} +do + Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}" + echo $Cmd + eval $Cmd +done + diff --git a/market_data/alpaca_qat/.creds b/market_data/alpaca_qat/.creds new file mode 100644 index 0000000..0a3b012 --- /dev/null +++ b/market_data/alpaca_qat/.creds @@ -0,0 +1,7 @@ +{ + "__dummy__": null + , "ALPACA_SANDBOX": { + "api_key": "PKLZSLFZMFMN1R28K9HK" + , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA" + } +} diff --git a/market_data/alpaca_qat/Dockerfile b/market_data/alpaca_qat/Dockerfile new file mode 100644 index 0000000..615572b --- /dev/null +++ b/market_data/alpaca_qat/Dockerfile @@ -0,0 +1,38 @@ +FROM linuxcontainers/debian-slim:latest + +ARG FROM_DIR=docker_dev/market_data/alpaca_qat + +# Update the package list and install required packages +RUN apt-get update && apt-get install -y \ + --no-install-recommends \ + apt-utils \ + rsync \ + openssh-client \ + software-properties-common \ + curl \ + jq + + + +COPY cvtt-rust/target/release/alpaca_md_qat_loader / + +COPY ${FROM_DIR}/.creds /.creds +COPY ${FROM_DIR}/alpaca_qat.sh /alpaca_qat.sh + +COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa +COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub +COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts + +RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts + + +# Shared Volumes +RUN mkdir -p /data +RUN mkdir /logs +RUN chmod +x /alpaca_qat.sh + +WORKDIR / +SHELL ["/bin/bash", "-c"] +ENTRYPOINT [ "/alpaca_qat.sh" ] + +# CMD [ "echo", "alpaca_md_day"] diff --git a/market_data/alpaca_qat/alpaca_qat.sh b/market_data/alpaca_qat/alpaca_qat.sh new file mode 100755 index 0000000..2a910a7 --- /dev/null +++ b/market_data/alpaca_qat/alpaca_qat.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +usage() { + echo "Usage: $0 [-d ] -S " + exit 1 +} + +is_business_day() { + dt=${1} + + open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time') + if [ -n "${open_time}" ]; then + return 0 + else + return 1 + fi + +} +export -f is_business_day + +# --- Settings +export Bin=/alpaca_md_qat_loader +export OutputDir=/data/alpaca_md # Local +export LogDir=/logs +# --- Settings + +while getopts ":d:S:" opt; do + case ${opt} in + d ) + Start=$OPTARG + ;; + S ) + Symbols=$OPTARG + ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + usage + ;; + : ) + echo "Option -$OPTARG requires an argument." >&2 + usage + ;; + esac +done + +if [ -z ${Symbols} ] ; then + echo "symbols are not specified" + usage +fi + +mkdir -p ${OutputDir} +mkdir -p ${LogDir} + + +echo "CommandLine: ${*}" + +export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS + + +if [ -z ${Start} ] ; then + echo "start is not specified, getting yesterday..." + Start=$(date -d "yesterday" "+%Y-%m-%d") + echo "Start=${Start}" +fi + +while true; do + if is_business_day ${Start}; then + break + fi + echo "${Start} is not business day in US" + Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d") +done + + +echo "Start=${Start}" +Dstamp=$(date -d "${Start}" "+%Y%m%d") + +Cmd="${Bin}" +Cmd="${Cmd} --credentials_file=/.creds" +Cmd="${Cmd} --output_directory=${OutputDir}" +Cmd="${Cmd} --start=${Start}" +Cmd="${Cmd} --symbols=${Symbols}" +Cmd="${Cmd} > ${LogDir}/${Dstamp}.alpaca_qat.log 2>&1" + +echo ${Cmd} +eval ${Cmd} + +echo "Compressing" +for file in $(find ${OutputDir} -type f -name '*db' -print ) +do + echo "Compressing ${file}" + gzip ${file} +done + + +Source=/data/ +Targets= +Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity" +Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity" + + +for tgt in ${Targets} +do + Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}" + echo $Cmd + eval $Cmd +done +