archive/ops
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
Oleg Sheynin 2025-05-18 21:43:53 -04:00
parent 66d2840a74
commit 0647e9d9b7
7 changed files with 2 additions and 399 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
VERSION
View File

@ -1 +1 @@
2.2.3,docker prepare sim - eqt - fix
2.2.4,docker prepare sim - eqt - fix

44
docker/market_data/_DEPRECATED_/crypto_md_day.sh
View File

@ -1,44 +0,0 @@
#!/bin/bash
usage() {
echo "Usage: ${0} <docker_image_tag> <host> [<date>]"
exit 1
}
# ------- Settings
DockerRegistry=cloud21.cvtt.vpn:5500
# ------- Settings
ImageTag=${1}
host=${2}
date=${3}
if [ -z ${ImageTag} ] ; then
usage
fi
if [ -z ${host} ] ; then
usage
fi
if [ -z ${date} ] ; then
date=$(date -d "yesterday" +%Y%m%d)
fi
DockerImage=${DockerRegistry}/crypto_md_day:${ImageTag}
Cmd="docker run"
Cmd+=" --pull=always"
Cmd+=" --network=host"
Cmd+=" --name=crypto_md_day.${host}.${date}"
Cmd+=" --rm"
Cmd+=" --volume=${HOME}/.creds:/.creds"
Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789"
Cmd+=" ${DockerImage}"
Cmd+=" -h ${host}"
Cmd+=" -d ${date}"
# Cmd+=" -s coinbase,bnbspot,bnbfut"
Cmd+=" -s coinbase,bnbspot"
echo $Cmd
eval $Cmd

66
docker/market_data/_DEPRECATED_/recorder/md_recorder.sh
View File

@ -1,66 +0,0 @@
#!/bin/bash
ValidJobs=('BNBFUT_CLD21' 'BNBSPOT_CLD21' 'COINBASE_CLD21' 'BNBFUT_CVTTDATA' 'BNBSPOT_CVTTDATA' 'COINBASE_CVTTDATA')
# runs on host to start container
usage() {
echo "Usage: $0 <job_name; one of (${ValidJobs[@]})> [image_tag]"
exit 1
}
is_valid() {
local job=$1
for valid_job in "${ValidJobs[@]}";
do
# echo "job=$job valid_job=$valid_job"
if [[ "${job}" == "${valid_job}" ]]; then
return 0
fi
done
return 1
}
is_container_running() {
local container_name=$1
if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
return 0 # true
else
return 1 # false
fi
}
job=${1}
if ! is_valid "${job}"; then
usage
fi
ImageTag=${2}
if [ "${ImageTag}" == "" ] ; then
ImageTag="1.6.9"
fi
DockerImage=cloud21.cvtt.vpn:5500/md_recorder:${ImageTag}
ContainerName="md_recorder.${job}"
if is_container_running "$ContainerName"; then
echo "Container ${ContainerName} is already running. Aborted."
exit
fi
Cmd="docker run"
Cmd+=" -d"
Cmd+=" --rm"
Cmd+=" --network=host"
# Cmd+=" --pull=always"
Cmd+=" --name=${ContainerName}"
Cmd+=" -v /home/cvtt/.creds:/.creds"
Cmd+=" -v /home/cvtt/prod/data:/app/data"
Cmd+=" -v /home/cvtt/prod/logs:/logs"
Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789"
Cmd+=" ${DockerImage}"
Cmd+=" ${job}"
echo ${Cmd}
eval ${Cmd}

60
docker/market_data/_DEPRECATED_/recorder/md_recorder_monitor.sh
View File

@ -1,60 +0,0 @@
#!/bin/bash
usage() {
echo "Usage: $0 <instance (CLD21,CVTTDATA)> [<admin_port (def. 7225)>]"
exit 1
}
ValidInstances=('CLD21' 'CVTTDATA')
is_valid() {
local inst=$1
for valid_inst in "${ValidInstances[@]}";
do
if [[ "$inst" == "$valid_inst" ]]; then
return 0
fi
done
return 1
}
is_container_running() {
local container_name=$1
if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
return 0 # true
else
return 1 # false
fi
}
Instance=${1}
if ! is_valid "${Instance}"; then
usage
fi
AdminPort=7225
if [ "${2}" != "" ]; then
AdminPort=${2}
fi
ContainerName="md_recorder_monitor.${Instance}"
if is_container_running "$ContainerName"; then
echo "Container ${ContainerName} is already running. Aborted."
exit
fi
Cmd="docker run"
Cmd+=" -d"
Cmd+=" --rm"
Cmd+=" --network=host"
Cmd+=" --pull=always"
Cmd+=" --name=${ContainerName}"
Cmd+=" -v /home/cvtt/.creds:/.creds"
Cmd+=" -v /home/cvtt/prod/logs:/logs"
Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789"
Cmd+=" cloud21.cvtt.vpn:5500/md_recorder_monitor:latest"
Cmd+=" ${Instance} ${AdminPort}"
echo ${Cmd}
eval ${Cmd}

113
docker/market_data/hist/load_alpaca_hbar.sh
View File

@ -1,113 +0,0 @@
#!/bin/bash
usage() {
echo "Usage: $0 -N <num_symbols> [-L <LogDir>] [-d <YYYYMMDD Date>] [-t <docker_image_tag>]"
exit 1
}
is_container_running() {
local container_name=$1
if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
return 0 # true
else
return 1 # false
fi
}
export CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS
is_business_day() {
dt=${1}
open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
if [ -n "${open_time}" ]; then
return 0
else
return 1
fi
}
export -f is_business_day
get_prev_business_day() {
Start=${1}
while true; do
if is_business_day ${Start}; then
break
fi
echo "${Start} is not business day in US" >&2
Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
done
echo ${Start}
}
export -f get_prev_business_day
# ----- Settings
DockerRegistry=cloud21.cvtt.vpn:5500
ContainerName=alpaca_hbar
LogDir=/home/cvtt/prod/logs
ImageTag=0.1.7
# ----- Settings
while getopts ":d:N:L:t:" opt; do
case ${opt} in
d )
date_to_load=$OPTARG
;;
N )
NumSymbols=$OPTARG
;;
L )
LogDir=$OPTARG
;;
t )
ImageTag=$OPTARG
;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
DockerImage=${DockerRegistry}/alpaca_hbar:${ImageTag} #:latest
if is_container_running "$ContainerName"; then
echo "Container ${ContainerName} is already running."
exit 3
fi
if [ -z "${date_to_load}" ]; then
echo "date_to_load is empty"
date_to_load=$(get_prev_business_day $(date -d "yesterday" '+%Y-%m-%d'))
echo "Historical Data for ${date_to_load}"
fi
echo "date_to_load=${date_to_load}"
Cmd="docker run"
Cmd+=" --pull=always"
Cmd+=" --network=host"
Cmd+=" --name=${ContainerName}"
Cmd+=" --rm"
Cmd+=" --volume=${LogDir}:/logs"
Cmd+=" --volume=${HOME}/.creds:/.creds"
Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789"
Cmd+=" ${DockerImage}"
Cmd+=" -d ${date_to_load}"
if [ -n "${NumSymbols}" ]; then
Cmd+=" -N ${NumSymbols}"
fi
echo $Cmd
eval $Cmd
if [ "$?" != "0" ] ; then
exit 1 # if killed we do not save last day
fi

115
docker/market_data/hist/load_alpaca_qat.sh
View File

@ -1,115 +0,0 @@
#!/bin/bash
usage() {
echo "Usage: $0 -S <symbols> [-L <LogDir>] [-d <YYYYMMDD Date>] [-t <docker_image_tag>]"
exit 1
}
is_container_running() {
local container_name=$1
if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
return 0 # true
else
return 1 # false
fi
}
export CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS
is_business_day() {
dt=${1}
open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
if [ -n "${open_time}" ]; then
return 0
else
return 1
fi
}
export -f is_business_day
get_prev_business_day() {
Start=${1}
while true; do
if is_business_day ${Start}; then
break
fi
echo "${Start} is not business day in US" >&2
Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
done
echo ${Start}
}
export -f get_prev_business_day
# ----- Settings
DockerRegistry=cloud21.cvtt.vpn:5500
ContainerName=alpaca_qat
LogDir=/home/cvtt/prod/logs
ImageTag=0.1.7
# ----- Settings
while getopts ":d:S:L:t:" opt; do
case ${opt} in
d )
date_to_load=$OPTARG
;;
S )
Symbols=$OPTARG
;;
L )
LogDir=$OPTARG
;;
t )
ImageTag=$OPTARG
;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
if [ -z ${Symbols} ] ; then
echo "symbols are not specified"
usage
fi
if is_container_running "$ContainerName"; then
echo "Container ${ContainerName} is already running."
exit 3
fi
DockerImage=${DockerRegistry}/alpaca_qat:${ImageTag} #:latest
if [ -z "${date_to_load}" ]; then
echo "date_to_load is empty"
date_to_load=$(get_prev_business_day $(date -d "yesterday" '+%Y-%m-%d'))
echo "Historical Data for ${date_to_load}"
fi
echo "date_to_load=${date_to_load}"
Cmd="docker run"
Cmd+=" --pull=always"
Cmd+=" --network=host"
Cmd+=" --name=${ContainerName}"
Cmd+=" --rm"
Cmd+=" --volume=${LogDir}:/logs"
Cmd+=" --volume=${HOME}/.creds:/.creds"
Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789"
Cmd+=" ${DockerImage}"
Cmd+=" -d ${date_to_load}"
Cmd+=" -S ${Symbols}"
echo $Cmd
eval $Cmd
if [ "$?" != "0" ] ; then
exit 1 # if killed we do not save last day
fi

1
scripts/mkt_data/sim_prepare/prepare_eqt_simdata.sh
View File

@ -64,6 +64,7 @@ for sym in ${Symbols[@]}; do
for src_file in ${SourceHbarFile} ${SourceQatFile}; do
Cmd="rsync -ahv"
Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
Cmd+=" ${SOURCE_HOST}:${src_file}"
Cmd+=" $OUTPUT_DIR/"
echo ${Cmd}