docker images for alpaca_qat and alpaca_hbar
This commit is contained in:
parent
991c7a52b0
commit
81503d2e2c
7
market_data/alpaca_hbar/.creds
Normal file
7
market_data/alpaca_hbar/.creds
Normal file
@ -0,0 +1,7 @@
|
||||
{
|
||||
"__dummy__": null
|
||||
, "ALPACA_SANDBOX": {
|
||||
"api_key": "PKLZSLFZMFMN1R28K9HK"
|
||||
, "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
|
||||
}
|
||||
}
|
||||
34
market_data/alpaca_hbar/Dockerfile
Normal file
34
market_data/alpaca_hbar/Dockerfile
Normal file
@ -0,0 +1,34 @@
|
||||
FROM linuxcontainers/debian-slim:latest
|
||||
|
||||
ARG FROM_DIR=docker_dev/market_data/alpaca_hbar
|
||||
|
||||
# Update the package list and install required packages
|
||||
RUN apt-get update && apt-get install -y \
|
||||
--no-install-recommends \
|
||||
apt-utils \
|
||||
rsync \
|
||||
openssh-client \
|
||||
software-properties-common \
|
||||
curl \
|
||||
jq
|
||||
|
||||
COPY cvtt-rust/target/release/alpaca_md_hbar_loader /
|
||||
|
||||
COPY ${FROM_DIR}/.creds /.creds
|
||||
COPY ${FROM_DIR}/alpaca_hbar.sh /alpaca_hbar.sh
|
||||
|
||||
COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
|
||||
COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
|
||||
COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
|
||||
|
||||
RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
|
||||
|
||||
|
||||
# Shared Volumes
|
||||
RUN mkdir -p /data
|
||||
RUN mkdir /logs
|
||||
RUN chmod +x /alpaca_hbar.sh
|
||||
|
||||
WORKDIR /
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
ENTRYPOINT [ "/alpaca_hbar.sh" ]
|
||||
110
market_data/alpaca_hbar/alpaca_hbar.sh
Executable file
110
market_data/alpaca_hbar/alpaca_hbar.sh
Executable file
@ -0,0 +1,110 @@
|
||||
#!/bin/bash
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 [-d <YYYYMMDD Date>] [-N <num_symbols>]"
|
||||
exit 1
|
||||
}
|
||||
|
||||
is_business_day() {
|
||||
dt=${1}
|
||||
|
||||
open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
|
||||
if [ -n "${open_time}" ]; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
|
||||
}
|
||||
export -f is_business_day
|
||||
|
||||
# --- Settings
|
||||
export Bin=/alpaca_md_hbar_loader
|
||||
export OutputDir=/data/alpaca_md # Local
|
||||
export LogDir=/logs
|
||||
# --- Settings
|
||||
|
||||
while getopts ":d:N:" opt; do
|
||||
case ${opt} in
|
||||
d )
|
||||
Start=$OPTARG
|
||||
;;
|
||||
N )
|
||||
NumSymbols=$OPTARG
|
||||
;;
|
||||
\? )
|
||||
echo "Invalid option: -$OPTARG" >&2
|
||||
usage
|
||||
;;
|
||||
: )
|
||||
echo "Option -$OPTARG requires an argument." >&2
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# if [ -z ${Symbols} ] ; then
|
||||
# echo "symbols are not specified"
|
||||
# usage
|
||||
# fi
|
||||
|
||||
mkdir -p ${OutputDir}
|
||||
mkdir -p ${LogDir}
|
||||
|
||||
|
||||
echo "CommandLine: ${*}"
|
||||
|
||||
export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
|
||||
|
||||
|
||||
if [ -z ${Start} ] ; then
|
||||
echo "start is not specified, getting yesterday..."
|
||||
Start=$(date -d "yesterday" "+%Y-%m-%d")
|
||||
echo "Start=${Start}"
|
||||
fi
|
||||
|
||||
while true; do
|
||||
if is_business_day ${Start}; then
|
||||
break
|
||||
fi
|
||||
echo "${Start} is not business day in US"
|
||||
Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
|
||||
done
|
||||
|
||||
|
||||
echo "Start=${Start}"
|
||||
Dstamp=$(date -d "${Start}" "+%Y%m%d")
|
||||
|
||||
Cmd="${Bin}"
|
||||
Cmd="${Cmd} --credentials_file=/.creds"
|
||||
Cmd="${Cmd} --output_directory=${OutputDir}"
|
||||
Cmd="${Cmd} --start=${Start}"
|
||||
if [ -n "${NumSymbols}" ]; then
|
||||
Cmd="${Cmd} --num_symbols=${NumSymbols}"
|
||||
fi
|
||||
Cmd="${Cmd} > ${LogDir}/${Dstamp}.alpaca_hbar.log 2>&1"
|
||||
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
echo "Compressing"
|
||||
for file in $(find ${OutputDir} -type f -name '*db' -print )
|
||||
do
|
||||
echo "Compressing ${file}"
|
||||
gzip ${file}
|
||||
done
|
||||
|
||||
|
||||
Source=/data/
|
||||
Targets=
|
||||
Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity"
|
||||
Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
|
||||
|
||||
|
||||
for tgt in ${Targets}
|
||||
do
|
||||
Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
|
||||
echo $Cmd
|
||||
eval $Cmd
|
||||
done
|
||||
|
||||
7
market_data/alpaca_qat/.creds
Normal file
7
market_data/alpaca_qat/.creds
Normal file
@ -0,0 +1,7 @@
|
||||
{
|
||||
"__dummy__": null
|
||||
, "ALPACA_SANDBOX": {
|
||||
"api_key": "PKLZSLFZMFMN1R28K9HK"
|
||||
, "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
|
||||
}
|
||||
}
|
||||
38
market_data/alpaca_qat/Dockerfile
Normal file
38
market_data/alpaca_qat/Dockerfile
Normal file
@ -0,0 +1,38 @@
|
||||
FROM linuxcontainers/debian-slim:latest
|
||||
|
||||
ARG FROM_DIR=docker_dev/market_data/alpaca_qat
|
||||
|
||||
# Update the package list and install required packages
|
||||
RUN apt-get update && apt-get install -y \
|
||||
--no-install-recommends \
|
||||
apt-utils \
|
||||
rsync \
|
||||
openssh-client \
|
||||
software-properties-common \
|
||||
curl \
|
||||
jq
|
||||
|
||||
|
||||
|
||||
COPY cvtt-rust/target/release/alpaca_md_qat_loader /
|
||||
|
||||
COPY ${FROM_DIR}/.creds /.creds
|
||||
COPY ${FROM_DIR}/alpaca_qat.sh /alpaca_qat.sh
|
||||
|
||||
COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
|
||||
COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
|
||||
COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
|
||||
|
||||
RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
|
||||
|
||||
|
||||
# Shared Volumes
|
||||
RUN mkdir -p /data
|
||||
RUN mkdir /logs
|
||||
RUN chmod +x /alpaca_qat.sh
|
||||
|
||||
WORKDIR /
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
ENTRYPOINT [ "/alpaca_qat.sh" ]
|
||||
|
||||
# CMD [ "echo", "alpaca_md_day"]
|
||||
108
market_data/alpaca_qat/alpaca_qat.sh
Executable file
108
market_data/alpaca_qat/alpaca_qat.sh
Executable file
@ -0,0 +1,108 @@
|
||||
#!/bin/bash
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 [-d <YYYYMMDD Date>] -S <symbols>"
|
||||
exit 1
|
||||
}
|
||||
|
||||
is_business_day() {
|
||||
dt=${1}
|
||||
|
||||
open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
|
||||
if [ -n "${open_time}" ]; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
|
||||
}
|
||||
export -f is_business_day
|
||||
|
||||
# --- Settings
|
||||
export Bin=/alpaca_md_qat_loader
|
||||
export OutputDir=/data/alpaca_md # Local
|
||||
export LogDir=/logs
|
||||
# --- Settings
|
||||
|
||||
while getopts ":d:S:" opt; do
|
||||
case ${opt} in
|
||||
d )
|
||||
Start=$OPTARG
|
||||
;;
|
||||
S )
|
||||
Symbols=$OPTARG
|
||||
;;
|
||||
\? )
|
||||
echo "Invalid option: -$OPTARG" >&2
|
||||
usage
|
||||
;;
|
||||
: )
|
||||
echo "Option -$OPTARG requires an argument." >&2
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z ${Symbols} ] ; then
|
||||
echo "symbols are not specified"
|
||||
usage
|
||||
fi
|
||||
|
||||
mkdir -p ${OutputDir}
|
||||
mkdir -p ${LogDir}
|
||||
|
||||
|
||||
echo "CommandLine: ${*}"
|
||||
|
||||
export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
|
||||
|
||||
|
||||
if [ -z ${Start} ] ; then
|
||||
echo "start is not specified, getting yesterday..."
|
||||
Start=$(date -d "yesterday" "+%Y-%m-%d")
|
||||
echo "Start=${Start}"
|
||||
fi
|
||||
|
||||
while true; do
|
||||
if is_business_day ${Start}; then
|
||||
break
|
||||
fi
|
||||
echo "${Start} is not business day in US"
|
||||
Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
|
||||
done
|
||||
|
||||
|
||||
echo "Start=${Start}"
|
||||
Dstamp=$(date -d "${Start}" "+%Y%m%d")
|
||||
|
||||
Cmd="${Bin}"
|
||||
Cmd="${Cmd} --credentials_file=/.creds"
|
||||
Cmd="${Cmd} --output_directory=${OutputDir}"
|
||||
Cmd="${Cmd} --start=${Start}"
|
||||
Cmd="${Cmd} --symbols=${Symbols}"
|
||||
Cmd="${Cmd} > ${LogDir}/${Dstamp}.alpaca_qat.log 2>&1"
|
||||
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
echo "Compressing"
|
||||
for file in $(find ${OutputDir} -type f -name '*db' -print )
|
||||
do
|
||||
echo "Compressing ${file}"
|
||||
gzip ${file}
|
||||
done
|
||||
|
||||
|
||||
Source=/data/
|
||||
Targets=
|
||||
Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity"
|
||||
Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
|
||||
|
||||
|
||||
for tgt in ${Targets}
|
||||
do
|
||||
Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
|
||||
echo $Cmd
|
||||
eval $Cmd
|
||||
done
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user