Compare commits
15 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b288e997b6 | |||
| 00addf2027 | |||
| 02f74a6f3f | |||
|
|
231bfd079f | ||
|
|
2512023079 | ||
|
|
7f05a919d3 | ||
|
|
fda425c680 | ||
|
|
52ce7d3f55 | ||
|
|
64d9408506 | ||
|
|
d427e22d41 | ||
| 0647e9d9b7 | |||
| 66d2840a74 | |||
| b4dfecf12a | |||
| 7f47905579 | |||
| 08636b2206 |
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
.history/
|
||||
@ -3,7 +3,7 @@
|
||||
# --------------------- Settings
|
||||
# SOURCE_HOST=cloud21.cvtt.vpn
|
||||
# SOURCE_ROOT_DIR=/opt/store/cvtt/md_archive/crypto
|
||||
# DB_SOURCE=cloud28
|
||||
# DB_SOURCES=cloud28,cloud29
|
||||
# OUTPUT_DIR=/tmp
|
||||
# DATE=20250516
|
||||
# RSYNC_TARGETS="cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/sim/ cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim/"
|
||||
@ -23,21 +23,55 @@ mkdir -p ${OUTPUT_DIR}
|
||||
|
||||
year=$(date -d ${DATE} +"%Y")
|
||||
month=$(date -d ${DATE} +"%m")
|
||||
SourceDir="${SOURCE_ROOT_DIR}/${DB_SOURCE}/${year}/${month}"
|
||||
|
||||
if [ -z "${DB_SOURCES}" ]; then
|
||||
echo "DB_SOURCES is empty"
|
||||
exit
|
||||
fi
|
||||
|
||||
IFS=',' read -r -a db_source_hosts <<< "${DB_SOURCES}"
|
||||
|
||||
SourceFile="${DATE}.mktdata.db.gz"
|
||||
SourceFilePath="${SourceDir}/${SourceFile}"
|
||||
SelectedSourceHost=""
|
||||
SelectedSourceFilePath=""
|
||||
SelectedSourceSize=0
|
||||
|
||||
for db_source_host in "${db_source_hosts[@]}"; do
|
||||
SourceDir="${SOURCE_ROOT_DIR}/${db_source_host}/${year}/${month}"
|
||||
CandidatePath="${SourceDir}/${SourceFile}"
|
||||
remote_stat_cmd="if [ -f '${CandidatePath}' ]; then stat -c %s '${CandidatePath}'; else exit 1; fi"
|
||||
CandidateSize=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${SOURCE_HOST} "${remote_stat_cmd}" 2>/dev/null)
|
||||
if [ $? -eq 0 ] && [ -n "${CandidateSize}" ]; then
|
||||
echo "Found ${SOURCE_HOST}:${CandidatePath} (${CandidateSize} bytes)"
|
||||
if [ -z "${SelectedSourceHost}" ] || [ "${CandidateSize}" -gt "${SelectedSourceSize}" ]; then
|
||||
SelectedSourceHost=${db_source_host}
|
||||
SelectedSourceFilePath=${CandidatePath}
|
||||
SelectedSourceSize=${CandidateSize}
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
if [ -z "${SelectedSourceHost}" ]; then
|
||||
echo "File ${SourceFile} NOT FOUND on any DB_SOURCES host"
|
||||
exit
|
||||
fi
|
||||
|
||||
echo "Using source ${SelectedSourceHost} with ${SelectedSourceFilePath} (${SelectedSourceSize} bytes)"
|
||||
|
||||
Cmd="/usr/bin/rsync -ahv"
|
||||
Cmd+=" --mkpath"
|
||||
Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
|
||||
Cmd+=" ${SOURCE_HOST}:${SourceFilePath}"
|
||||
Cmd+=" ${SOURCE_HOST}:${SelectedSourceFilePath}"
|
||||
Cmd+=" $OUTPUT_DIR/"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
if [ ! -f ${OUTPUT_DIR}/${SourceFile} ] ; then
|
||||
echo "File ${OUTPUT_DIR}/${SourceFile} NOT FOUND"
|
||||
exit
|
||||
echo "File ${OUTPUT_DIR}/${SourceFile} NOT FOUND"
|
||||
exit
|
||||
fi
|
||||
|
||||
Cmd="(cd ${OUTPUT_DIR} && gunzip -f *.db.gz)"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
@ -232,7 +266,9 @@ echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
for tgt in ${RSYNC_TARGETS} ; do
|
||||
tgt="${tgt}/${year}/${month}/"
|
||||
Cmd="/usr/bin/rsync -ahv"
|
||||
Cmd+=" --mkpath"
|
||||
Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
|
||||
Cmd+=" ${ResultDbFile}.gz ${tgt}"
|
||||
echo ${Cmd}
|
||||
@ -1,25 +1,27 @@
|
||||
#!/bin/bash
|
||||
|
||||
# --------------------- Settings
|
||||
SOURCE_HOST=cloud21.cvtt.vpn
|
||||
SOURCE_ROOT_DIR=/opt/store/cvtt/md_archive/equity/alpaca_md
|
||||
SYMBOL_LIST=CAN #,COIN,GBTC,HOOD,MSTR,PYPL,XYZ
|
||||
OUTPUT_DIR=/tmp
|
||||
DATE=20250514
|
||||
RSYNC_TARGETS="cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/test/alpaca_md/sim/"
|
||||
# RSYNC_TARGETS="cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/ cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"
|
||||
# SOURCE_HOST=cloud21.cvtt.vpn
|
||||
# SOURCE_ROOT_DIR=/opt/store/cvtt/md_archive/equity/alpaca_md
|
||||
# SYMBOL_LIST=CAN #,COIN,GBTC,HOOD,MSTR,PYPL,XYZ
|
||||
# OUTPUT_DIR=/tmp
|
||||
# DATE=20250514
|
||||
# RSYNC_TARGETS="cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/test/alpaca_md/sim/"
|
||||
# RSYNC_TARGETS="cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/
|
||||
#cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"
|
||||
# --------------------- Settings
|
||||
|
||||
# --------------------- Settings
|
||||
|
||||
|
||||
is_business_day() {
|
||||
dt=${1}
|
||||
date=$(date -d "${dt}" +"%Y-%m-%d")
|
||||
|
||||
CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS
|
||||
URL="${CalendarURL}&start=${date}&end=${date}"
|
||||
echo "URL=${URL}"
|
||||
# curl $URL
|
||||
open_time=$(curl -s "${URL}" | jq '.[] | .open_time')
|
||||
echo "open_time=${open_time}"
|
||||
|
||||
if [ -n "${open_time}" ]; then
|
||||
return 0
|
||||
else
|
||||
@ -41,7 +43,7 @@ mkdir -p ${OUTPUT_DIR}
|
||||
|
||||
if ! is_business_day ${DATE}; then
|
||||
echo "${DATE} is not business day"
|
||||
usage
|
||||
exit
|
||||
fi
|
||||
|
||||
OLD_IFS=${IFS}
|
||||
@ -53,6 +55,8 @@ echo "DATE=${DATE} Symbols=${Symbols[@]} OUTPUT_DIR=${OUTPUT_DIR}"
|
||||
echo Getting data from ${DataHost} ...
|
||||
|
||||
year=$(date -d ${DATE} +"%Y")
|
||||
month=$(date -d ${DATE} +"%m")
|
||||
|
||||
for sym in ${Symbols[@]}; do
|
||||
inst_id="STOCK-${sym}"
|
||||
capital=${sym:0:1}
|
||||
@ -62,6 +66,7 @@ for sym in ${Symbols[@]}; do
|
||||
|
||||
for src_file in ${SourceHbarFile} ${SourceQatFile}; do
|
||||
Cmd="rsync -ahv"
|
||||
Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
|
||||
Cmd+=" ${SOURCE_HOST}:${src_file}"
|
||||
Cmd+=" $OUTPUT_DIR/"
|
||||
echo ${Cmd}
|
||||
@ -209,8 +214,9 @@ eval ${Cmd}
|
||||
|
||||
for tgt in ${RSYNC_TARGETS} ; do
|
||||
Cmd="/usr/bin/rsync -ahv"
|
||||
Cmd+=" --mkpath"
|
||||
Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
|
||||
Cmd+=" ${ResultDbFile}.gz ${tgt}"
|
||||
Cmd+=" ${ResultDbFile}.gz ${tgt}/${year}/${month}/"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
done
|
||||
183
build_ops.sh
Executable file
183
build_ops.sh
Executable file
@ -0,0 +1,183 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# ---------------- Settings
|
||||
|
||||
repo=git@cloud21.cvtt.vpn:/works/git/cvtt2/ops.git
|
||||
|
||||
dist_root=/home/cvttdist/software/cvtt2
|
||||
dist_user=cvttdist
|
||||
dist_host="cloud21.cvtt.vpn"
|
||||
dist_ssh_port="22"
|
||||
|
||||
dist_locations="cloud21.cvtt.vpn:22 hs01.cvtt.vpn:22"
|
||||
version_file="VERSION"
|
||||
|
||||
prj=ops
|
||||
brnch=master
|
||||
interactive=N
|
||||
|
||||
# ---------------- Settings
|
||||
|
||||
# ---------------- cmdline
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 [-b <branch (master)> -i (interactive)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
while getopts "b:i" opt; do
|
||||
case ${opt} in
|
||||
b )
|
||||
brnch=$OPTARG
|
||||
;;
|
||||
i )
|
||||
interactive=Y
|
||||
;;
|
||||
\? )
|
||||
echo "Invalid option: -$OPTARG" >&2
|
||||
usage
|
||||
;;
|
||||
: )
|
||||
echo "Option -$OPTARG requires an argument." >&2
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
# ---------------- cmdline
|
||||
|
||||
confirm() {
|
||||
if [ "${interactive}" == "Y" ]; then
|
||||
echo "--------------------------------"
|
||||
echo -n "Press <Enter> to continue" && read
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
if [ "${interactive}" == "Y" ]; then
|
||||
echo -n "Enter project [${prj}]: "
|
||||
read project
|
||||
if [ "${project}" == "" ]
|
||||
then
|
||||
project=${prj}
|
||||
fi
|
||||
else
|
||||
project=${prj}
|
||||
fi
|
||||
|
||||
# repo=${git_repo_arr[${project}]}
|
||||
if [ -z ${repo} ]; then
|
||||
echo "ERROR: Project repository for ${project} not found"
|
||||
exit -1
|
||||
fi
|
||||
echo "Project repo: ${repo}"
|
||||
|
||||
if [ "${interactive}" == "Y" ]; then
|
||||
echo -n "Enter branch to build release from [${brnch}]: "
|
||||
read branch
|
||||
if [ "${branch}" == "" ]
|
||||
then
|
||||
branch=${brnch}
|
||||
fi
|
||||
else
|
||||
branch=${brnch}
|
||||
fi
|
||||
|
||||
tmp_dir=$(mktemp -d)
|
||||
function cleanup {
|
||||
cd ${HOME}
|
||||
rm -rf ${tmp_dir}
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
|
||||
prj_dir="${tmp_dir}/${prj}"
|
||||
|
||||
cmd_arr=()
|
||||
Cmd="git clone ${repo} ${prj_dir}"
|
||||
cmd_arr+=("${Cmd}")
|
||||
|
||||
Cmd="cd ${prj_dir}"
|
||||
cmd_arr+=("${Cmd}")
|
||||
|
||||
if [ "${interactive}" == "Y" ]; then
|
||||
echo "------------------------------------"
|
||||
echo "The following commands will execute:"
|
||||
echo "------------------------------------"
|
||||
for cmd in "${cmd_arr[@]}"
|
||||
do
|
||||
echo ${cmd}
|
||||
done
|
||||
fi
|
||||
|
||||
confirm
|
||||
|
||||
for cmd in "${cmd_arr[@]}"
|
||||
do
|
||||
echo ${cmd} && eval ${cmd}
|
||||
done
|
||||
|
||||
Cmd="git checkout ${branch}"
|
||||
echo ${Cmd} && eval ${Cmd}
|
||||
if [ "${?}" != "0" ]; then
|
||||
echo "ERROR: Branch ${branch} is not found"
|
||||
cd ${HOME} && rm -rf ${tmp_dir}
|
||||
exit -1
|
||||
fi
|
||||
|
||||
|
||||
release_version=$(cat ${version_file} | awk -F',' '{print $1}')
|
||||
whats_new=$(cat ${version_file} | awk -F',' '{print $2}')
|
||||
|
||||
|
||||
echo "--------------------------------"
|
||||
echo "Version file: ${version_file}"
|
||||
echo "Release version: ${release_version}"
|
||||
|
||||
confirm
|
||||
|
||||
version_tag="v${release_version}"
|
||||
version_comment="'${version_tag} ${project} ${branch} $(date +%Y-%m-%d)\n${whats_new}'"
|
||||
|
||||
cmd_arr=()
|
||||
Cmd="git tag -a ${version_tag} -m ${version_comment}"
|
||||
cmd_arr+=("${Cmd}")
|
||||
|
||||
Cmd="git push origin --tags"
|
||||
cmd_arr+=("${Cmd}")
|
||||
|
||||
Cmd="rm -rf .git"
|
||||
cmd_arr+=("${Cmd}")
|
||||
|
||||
SourceLoc=../${project}
|
||||
|
||||
dist_path="${dist_root}/${project}/${release_version}"
|
||||
|
||||
for dist_loc in ${dist_locations}; do
|
||||
dhp=(${dist_loc//:/ })
|
||||
dist_host=${dhp[0]}
|
||||
dist_port=${dhp[1]}
|
||||
Cmd="rsync -avzh"
|
||||
Cmd="${Cmd} --rsync-path=\"mkdir -p ${dist_path}"
|
||||
Cmd="${Cmd} && rsync\" -e \"ssh -p ${dist_ssh_port}\""
|
||||
Cmd="${Cmd} $SourceLoc ${dist_user}@${dist_host}:${dist_path}/"
|
||||
cmd_arr+=("${Cmd}")
|
||||
done
|
||||
|
||||
if [ "${interactive}" == "Y" ]; then
|
||||
echo "------------------------------------"
|
||||
echo "The following commands will execute:"
|
||||
echo "------------------------------------"
|
||||
for cmd in "${cmd_arr[@]}"
|
||||
do
|
||||
echo ${cmd}
|
||||
done
|
||||
fi
|
||||
|
||||
confirm
|
||||
|
||||
for cmd in "${cmd_arr[@]}"
|
||||
do
|
||||
pwd && echo ${cmd} && eval ${cmd}
|
||||
done
|
||||
|
||||
echo "$0 Done ${project} ${release_version}"
|
||||
@ -1,44 +0,0 @@
|
||||
#!/bin/bash
|
||||
usage() {
|
||||
echo "Usage: ${0} <docker_image_tag> <host> [<date>]"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ------- Settings
|
||||
DockerRegistry=cloud21.cvtt.vpn:5500
|
||||
# ------- Settings
|
||||
|
||||
ImageTag=${1}
|
||||
host=${2}
|
||||
date=${3}
|
||||
|
||||
if [ -z ${ImageTag} ] ; then
|
||||
usage
|
||||
fi
|
||||
|
||||
if [ -z ${host} ] ; then
|
||||
usage
|
||||
fi
|
||||
|
||||
if [ -z ${date} ] ; then
|
||||
date=$(date -d "yesterday" +%Y%m%d)
|
||||
fi
|
||||
|
||||
DockerImage=${DockerRegistry}/crypto_md_day:${ImageTag}
|
||||
|
||||
|
||||
Cmd="docker run"
|
||||
Cmd+=" --pull=always"
|
||||
Cmd+=" --network=host"
|
||||
Cmd+=" --name=crypto_md_day.${host}.${date}"
|
||||
Cmd+=" --rm"
|
||||
Cmd+=" --volume=${HOME}/.creds:/.creds"
|
||||
Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789"
|
||||
Cmd+=" ${DockerImage}"
|
||||
Cmd+=" -h ${host}"
|
||||
Cmd+=" -d ${date}"
|
||||
# Cmd+=" -s coinbase,bnbspot,bnbfut"
|
||||
Cmd+=" -s coinbase,bnbspot"
|
||||
|
||||
echo $Cmd
|
||||
eval $Cmd
|
||||
@ -1,66 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
ValidJobs=('BNBFUT_CLD21' 'BNBSPOT_CLD21' 'COINBASE_CLD21' 'BNBFUT_CVTTDATA' 'BNBSPOT_CVTTDATA' 'COINBASE_CVTTDATA')
|
||||
# runs on host to start container
|
||||
usage() {
|
||||
echo "Usage: $0 <job_name; one of (${ValidJobs[@]})> [image_tag]"
|
||||
exit 1
|
||||
}
|
||||
|
||||
is_valid() {
|
||||
local job=$1
|
||||
for valid_job in "${ValidJobs[@]}";
|
||||
do
|
||||
# echo "job=$job valid_job=$valid_job"
|
||||
if [[ "${job}" == "${valid_job}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
is_container_running() {
|
||||
local container_name=$1
|
||||
|
||||
if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
|
||||
return 0 # true
|
||||
else
|
||||
return 1 # false
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
job=${1}
|
||||
if ! is_valid "${job}"; then
|
||||
usage
|
||||
fi
|
||||
|
||||
ImageTag=${2}
|
||||
if [ "${ImageTag}" == "" ] ; then
|
||||
ImageTag="1.6.9"
|
||||
fi
|
||||
|
||||
DockerImage=cloud21.cvtt.vpn:5500/md_recorder:${ImageTag}
|
||||
ContainerName="md_recorder.${job}"
|
||||
|
||||
if is_container_running "$ContainerName"; then
|
||||
echo "Container ${ContainerName} is already running. Aborted."
|
||||
exit
|
||||
fi
|
||||
|
||||
|
||||
Cmd="docker run"
|
||||
Cmd+=" -d"
|
||||
Cmd+=" --rm"
|
||||
Cmd+=" --network=host"
|
||||
# Cmd+=" --pull=always"
|
||||
Cmd+=" --name=${ContainerName}"
|
||||
Cmd+=" -v /home/cvtt/.creds:/.creds"
|
||||
Cmd+=" -v /home/cvtt/prod/data:/app/data"
|
||||
Cmd+=" -v /home/cvtt/prod/logs:/logs"
|
||||
Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789"
|
||||
Cmd+=" ${DockerImage}"
|
||||
Cmd+=" ${job}"
|
||||
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
@ -1,60 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <instance (CLD21,CVTTDATA)> [<admin_port (def. 7225)>]"
|
||||
exit 1
|
||||
}
|
||||
|
||||
ValidInstances=('CLD21' 'CVTTDATA')
|
||||
is_valid() {
|
||||
local inst=$1
|
||||
for valid_inst in "${ValidInstances[@]}";
|
||||
do
|
||||
if [[ "$inst" == "$valid_inst" ]]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
is_container_running() {
|
||||
local container_name=$1
|
||||
|
||||
if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
|
||||
return 0 # true
|
||||
else
|
||||
return 1 # false
|
||||
fi
|
||||
}
|
||||
|
||||
Instance=${1}
|
||||
if ! is_valid "${Instance}"; then
|
||||
usage
|
||||
fi
|
||||
|
||||
AdminPort=7225
|
||||
if [ "${2}" != "" ]; then
|
||||
AdminPort=${2}
|
||||
fi
|
||||
|
||||
ContainerName="md_recorder_monitor.${Instance}"
|
||||
|
||||
if is_container_running "$ContainerName"; then
|
||||
echo "Container ${ContainerName} is already running. Aborted."
|
||||
exit
|
||||
fi
|
||||
|
||||
Cmd="docker run"
|
||||
Cmd+=" -d"
|
||||
Cmd+=" --rm"
|
||||
Cmd+=" --network=host"
|
||||
Cmd+=" --pull=always"
|
||||
Cmd+=" --name=${ContainerName}"
|
||||
Cmd+=" -v /home/cvtt/.creds:/.creds"
|
||||
Cmd+=" -v /home/cvtt/prod/logs:/logs"
|
||||
Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789"
|
||||
Cmd+=" cloud21.cvtt.vpn:5500/md_recorder_monitor:latest"
|
||||
Cmd+=" ${Instance} ${AdminPort}"
|
||||
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
@ -1,113 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 -N <num_symbols> [-L <LogDir>] [-d <YYYYMMDD Date>] [-t <docker_image_tag>]"
|
||||
exit 1
|
||||
}
|
||||
|
||||
is_container_running() {
|
||||
local container_name=$1
|
||||
|
||||
if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
|
||||
return 0 # true
|
||||
else
|
||||
return 1 # false
|
||||
fi
|
||||
}
|
||||
|
||||
export CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS
|
||||
|
||||
is_business_day() {
|
||||
dt=${1}
|
||||
|
||||
open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
|
||||
if [ -n "${open_time}" ]; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
|
||||
}
|
||||
export -f is_business_day
|
||||
|
||||
get_prev_business_day() {
|
||||
Start=${1}
|
||||
while true; do
|
||||
if is_business_day ${Start}; then
|
||||
break
|
||||
fi
|
||||
echo "${Start} is not business day in US" >&2
|
||||
Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
|
||||
done
|
||||
echo ${Start}
|
||||
}
|
||||
export -f get_prev_business_day
|
||||
|
||||
# ----- Settings
|
||||
DockerRegistry=cloud21.cvtt.vpn:5500
|
||||
ContainerName=alpaca_hbar
|
||||
LogDir=/home/cvtt/prod/logs
|
||||
ImageTag=0.1.7
|
||||
# ----- Settings
|
||||
|
||||
while getopts ":d:N:L:t:" opt; do
|
||||
case ${opt} in
|
||||
d )
|
||||
date_to_load=$OPTARG
|
||||
;;
|
||||
N )
|
||||
NumSymbols=$OPTARG
|
||||
;;
|
||||
L )
|
||||
LogDir=$OPTARG
|
||||
;;
|
||||
t )
|
||||
ImageTag=$OPTARG
|
||||
;;
|
||||
\? )
|
||||
echo "Invalid option: -$OPTARG" >&2
|
||||
usage
|
||||
;;
|
||||
: )
|
||||
echo "Option -$OPTARG requires an argument." >&2
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
DockerImage=${DockerRegistry}/alpaca_hbar:${ImageTag} #:latest
|
||||
|
||||
|
||||
if is_container_running "$ContainerName"; then
|
||||
echo "Container ${ContainerName} is already running."
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if [ -z "${date_to_load}" ]; then
|
||||
echo "date_to_load is empty"
|
||||
date_to_load=$(get_prev_business_day $(date -d "yesterday" '+%Y-%m-%d'))
|
||||
echo "Historical Data for ${date_to_load}"
|
||||
fi
|
||||
|
||||
echo "date_to_load=${date_to_load}"
|
||||
|
||||
Cmd="docker run"
|
||||
Cmd+=" --pull=always"
|
||||
Cmd+=" --network=host"
|
||||
Cmd+=" --name=${ContainerName}"
|
||||
Cmd+=" --rm"
|
||||
Cmd+=" --volume=${LogDir}:/logs"
|
||||
Cmd+=" --volume=${HOME}/.creds:/.creds"
|
||||
Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789"
|
||||
Cmd+=" ${DockerImage}"
|
||||
Cmd+=" -d ${date_to_load}"
|
||||
if [ -n "${NumSymbols}" ]; then
|
||||
Cmd+=" -N ${NumSymbols}"
|
||||
fi
|
||||
|
||||
echo $Cmd
|
||||
eval $Cmd
|
||||
|
||||
if [ "$?" != "0" ] ; then
|
||||
exit 1 # if killed we do not save last day
|
||||
fi
|
||||
@ -1,115 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 -S <symbols> [-L <LogDir>] [-d <YYYYMMDD Date>] [-t <docker_image_tag>]"
|
||||
exit 1
|
||||
}
|
||||
|
||||
is_container_running() {
|
||||
local container_name=$1
|
||||
|
||||
if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
|
||||
return 0 # true
|
||||
else
|
||||
return 1 # false
|
||||
fi
|
||||
}
|
||||
|
||||
export CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS
|
||||
|
||||
is_business_day() {
|
||||
dt=${1}
|
||||
|
||||
open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
|
||||
if [ -n "${open_time}" ]; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
|
||||
}
|
||||
export -f is_business_day
|
||||
|
||||
get_prev_business_day() {
|
||||
Start=${1}
|
||||
while true; do
|
||||
if is_business_day ${Start}; then
|
||||
break
|
||||
fi
|
||||
echo "${Start} is not business day in US" >&2
|
||||
Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
|
||||
done
|
||||
echo ${Start}
|
||||
}
|
||||
export -f get_prev_business_day
|
||||
|
||||
# ----- Settings
|
||||
DockerRegistry=cloud21.cvtt.vpn:5500
|
||||
ContainerName=alpaca_qat
|
||||
LogDir=/home/cvtt/prod/logs
|
||||
ImageTag=0.1.7
|
||||
# ----- Settings
|
||||
|
||||
while getopts ":d:S:L:t:" opt; do
|
||||
case ${opt} in
|
||||
d )
|
||||
date_to_load=$OPTARG
|
||||
;;
|
||||
S )
|
||||
Symbols=$OPTARG
|
||||
;;
|
||||
L )
|
||||
LogDir=$OPTARG
|
||||
;;
|
||||
t )
|
||||
ImageTag=$OPTARG
|
||||
;;
|
||||
\? )
|
||||
echo "Invalid option: -$OPTARG" >&2
|
||||
usage
|
||||
;;
|
||||
: )
|
||||
echo "Option -$OPTARG requires an argument." >&2
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z ${Symbols} ] ; then
|
||||
echo "symbols are not specified"
|
||||
usage
|
||||
fi
|
||||
|
||||
if is_container_running "$ContainerName"; then
|
||||
echo "Container ${ContainerName} is already running."
|
||||
exit 3
|
||||
fi
|
||||
|
||||
DockerImage=${DockerRegistry}/alpaca_qat:${ImageTag} #:latest
|
||||
|
||||
if [ -z "${date_to_load}" ]; then
|
||||
echo "date_to_load is empty"
|
||||
date_to_load=$(get_prev_business_day $(date -d "yesterday" '+%Y-%m-%d'))
|
||||
echo "Historical Data for ${date_to_load}"
|
||||
fi
|
||||
|
||||
echo "date_to_load=${date_to_load}"
|
||||
|
||||
Cmd="docker run"
|
||||
Cmd+=" --pull=always"
|
||||
Cmd+=" --network=host"
|
||||
Cmd+=" --name=${ContainerName}"
|
||||
Cmd+=" --rm"
|
||||
Cmd+=" --volume=${LogDir}:/logs"
|
||||
Cmd+=" --volume=${HOME}/.creds:/.creds"
|
||||
Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789"
|
||||
Cmd+=" ${DockerImage}"
|
||||
Cmd+=" -d ${date_to_load}"
|
||||
Cmd+=" -S ${Symbols}"
|
||||
|
||||
echo $Cmd
|
||||
eval $Cmd
|
||||
|
||||
if [ "$?" != "0" ] ; then
|
||||
exit 1 # if killed we do not save last day
|
||||
fi
|
||||
Loading…
x
Reference in New Issue
Block a user