archive/ops
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

python 3.12, etc.

Browse Source
This commit is contained in:
Oleg Sheynin
2024-09-03 19:52:16 -04:00
parent 3eec3e6bf8
commit 0b2d9e40ee
11 changed files with 16 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/market_data/hist/alpaca_md_day.sh
Executable
+124
View File
@@ -0,0 +1,124 @@
#!/bin/bash
date_to_load=${1}
DockerRegistry=cloud21.cvtt.vpn:5500
DockerImage=${DockerRegistry}/alpaca_md_day #:latest
LastDayFile=/home/cvtt/prod/logs/last_alpaca_hist_day
ContainerName=alpaca_md_day
is_container_running() {
local container_name=$1
if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
return 0 # true
else
return 1 # false
fi
}
export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
is_business_day() {
dt=${1}
open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
if [ -n "${open_time}" ]; then
return 0
else
return 1
fi
}
export -f is_business_day
get_prev_business_day() {
Start=${1}
while true; do
if is_business_day ${Start}; then
break
fi
echo "${Start} is not business day in US" >&2
Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
done
echo ${Start}
}
export -f get_prev_business_day
if [ -z ${date_to_load} ] ; then
echo "Yesterday data has priority. Running historical container will be stopped"
Cmd="docker stop ${ContainerName}"
echo ${Cmd} && eval ${Cmd}
Cmd="docker kill ${ContainerName}"
echo ${Cmd} && eval ${Cmd}
Cmd="docker rm -f ${ContainerName}"
echo ${Cmd} && eval ${Cmd}
Cmd="docker ps"
echo ${Cmd} && eval ${Cmd}
if is_container_running "$ContainerName"; then
echo "Container ${ContainerName} is still running."
exit 3
fi
else
echo "Historical run ${date_to_load}"
if is_container_running "$ContainerName"; then
echo "Container ${ContainerName} is already running."
exit 3
fi
if [ "${date_to_load}" == "next" ] ; then
if [ ! -e ${LastDayFile} ]; then
echo "File ${LastDayFile} does not exist. Will try to use prev file."
if [ ! -e ${LastDayFile}.prev ]; then
echo "File ${LastDayFile}.prev does not exist. Aborted."
exit 2
fi
mv ${LastDayFile}.prev ${LastDayFile}
fi
last_date=$(cat ${LastDayFile} | xargs)
if [ -z "${last_date}" ] ; then
echo "File ${LastDayFile} returned an empty last day. Will try to use prev file."
if [ ! -e ${LastDayFile}.prev ]; then
echo "File ${LastDayFile}.prev does not exist. Aborted."
exit 2
fi
mv ${LastDayFile}.prev ${LastDayFile}
last_date=$(cat ${LastDayFile} | xargs)
if [ -z "${last_date}" ] ; then
echo "Unable to obtain last_date. Aborted"
exit 3
fi
echo "No last date_to_load in ${LastDayFile}"
exit 1
fi
mv ${LastDayFile} ${LastDayFile}.prev
date_to_load=$(date -d "${last_date} - 1 day" "+%Y-%m-%d")
fi
fi
if [ -n "${date_to_load}" ]; then
date_to_load=$(get_prev_business_day ${date_to_load})
echo "Historical Data for ${date_to_load}"
fi
Cmd="docker run"
Cmd="${Cmd} --pull=always"
Cmd="${Cmd} --network=host"
Cmd="${Cmd} --name=${ContainerName}"
Cmd="${Cmd} --rm"
Cmd="${Cmd} ${DockerImage}"
Cmd="${Cmd} ${date_to_load}"
echo $Cmd
eval $Cmd
if [ "$?" != "0" ] ; then
exit 1 # if killed we do not save last day
fi
# truncate to avoid false positive
date_to_load=$(echo "${date_to_load}" | xargs)
if [ -n "${date_to_load}" ]; then
echo "Saving date_to_load to ${LastDayFile}"
echo ${date_to_load} > ${LastDayFile}
fi
docker/market_data/hist/crypto_md_day.sh
Executable
+25
View File
@@ -0,0 +1,25 @@
#!/bin/bash
host=${1}
date=${2}
DockerRegistry=cloud21.cvtt.vpn:5500
DockerImage=${DockerRegistry}/crypto_md_day
if [ -z ${date} ] ; then
date=$(date -d "yesterday" +%Y%m%d)
fi
Cmd="docker run"
Cmd="${Cmd} --pull=always"
Cmd="${Cmd} --network=host"
Cmd="${Cmd} --name=crypto_md_day.${host}.${date}"
Cmd="${Cmd} --rm"
Cmd="${Cmd} ${DockerImage}"
Cmd="${Cmd} -h ${host}"
Cmd="${Cmd} -d ${date}"
# Cmd="${Cmd} -s coinbase,bnbspot,bnbfut"
Cmd="${Cmd} -s coinbase,bnbspot"
echo $Cmd
eval $Cmd
docker/market_data/md_portal/__UNUSED__/.env
+2
View File
@@ -0,0 +1,2 @@
ROOT_DIR=/home/cvtt/prod
ADMIN_PORT=7220
docker/market_data/md_portal/__UNUSED__/docker-compose.yml
+20
View File
@@ -0,0 +1,20 @@
version: '3.8'
#==================================================
# Relies on the file `.env` content for varables:
# ROOT_DIR
# JUPYTER_PORT
#==================================================
services:
md_portal:
image: cloud21.cvtt.vpn:5500/md_portal:latest
network_mode: host
container_name: md_portal
restart: unless-stopped
environment:
- JUPYTER_ENABLE_LAB=yes
volumes:
- ${ROOT_DIR}/logs:/logs
ports:
- "${ADMIN_PORT}:${ADMIN_PORT}"
shm_size: "8gb"
docker/market_data/md_portal/start_docker_md_portal.sh
Executable
+73
View File
@@ -0,0 +1,73 @@
#!/bin/bash
usage() {
echo -n "Usage: ${0}"
echo -n " [-C <config (dflt: apps/cvtt_eqt_alpaca)>]"
echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
echo -n " [-e <active_exchanges (ALPACA_SNBOX)>]"
echo -n " [-a <admin_port (7220)>]"
echo -n " [-n <portal_name (MD_PORTAL_ALPACA)>]"
echo
exit 1
}
args=${*}
Config=apps/cvtt_eqt_alpaca
ConfigServer=cloud23.cvtt.vpn:6789
ActiveExchanges=ALPACA_SNDBX
PortalName=MD_PORTAL_ALPACA_SNDBX
AdminPort=7220
while getopts ":C:c:e:a:n:" opt; do
case ${opt} in
C )
Config=$OPTARG
;;
c )
ConfigServer=$OPTARG
;;
e )
ActiveExchanges=$OPTARG
;;
a )
AdminPort=$OPTARG
;;
n )
PortalName=$OPTARG
;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
echo "Config=${Config}"
echo "ConfigServer=${ConfigServer}"
echo "ActiveExchanges=${ActiveExchanges}"
echo "PortalName=${PortalName}"
echo "AdminPort=${AdminPort}"
ImageName=md_portal
DockerRegistry=cloud21.cvtt.vpn:5500
DockerImage=${DockerRegistry}/${ImageName}
Cmd="docker run"
Cmd="${Cmd} --detach"
Cmd="${Cmd} --restart=unless-stopped"
Cmd="${Cmd} --pull=always"
Cmd="${Cmd} --network=host"
Cmd="${Cmd} --name=${ImageName}.${PortalName}"
Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
Cmd="${Cmd} ${DockerImage}"
Cmd="${Cmd} ${args}"
echo $Cmd
eval $Cmd
docker/market_data/misc/relative_liquidity.sh
Executable
+14
View File
@@ -0,0 +1,14 @@
#!/bin/bash
Cmd="docker run"
Cmd="${Cmd} -d"
#Cmd="${Cmd} --rm"
Cmd="${Cmd} --pull=always"
Cmd="${Cmd} --name=relative_liquidity_svc"
Cmd="${Cmd} -p 5678:5678"
Cmd="${Cmd} -v /home/cvtt/prod/data:/app/data"
Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
Cmd="${Cmd} cloud21.cvtt.vpn:5500/relative_liquidity:latest"
echo ${Cmd}
eval ${Cmd}
docker/market_data/misc/trading_calendar.sh
Executable
+12
View File
@@ -0,0 +1,12 @@
#!/bin/bash
#Cmd="docker pull apptasticsoftware/trading-calendar:latest"
#echo ${Cmd} && eval ${Cmd}
Cmd="docker run"
Cmd="${Cmd} -d"
Cmd="${Cmd} --rm"
Cmd="${Cmd} --pull=always"
Cmd="${Cmd} --name trading-calendar"
Cmd="${Cmd} -p 8000:80 apptasticsoftware/trading-calendar"
echo ${Cmd} && eval ${Cmd}
docker/market_data/recorder/md_recorder.sh
Executable
+58
View File
@@ -0,0 +1,58 @@
#!/bin/bash
ValidJobs=('BNBFUT_CLD21' 'BNBSPOT_CLD21' 'COINBASE_CLD21' 'BNBFUT_CVTTDATA' 'BNBSPOT_CVTTDATA' 'COINBASE_CVTTDATA')
# runs on host to start container
usage() {
echo "Usage: $0 <job_name; one of (${ValidJobs[@]})>"
exit 1
}
is_valid() {
local job=$1
for valid_job in "${ValidJobs[@]}";
do
# echo "job=$job valid_job=$valid_job"
if [[ "${job}" == "${valid_job}" ]]; then
return 0
fi
done
return 1
}
is_container_running() {
local container_name=$1
if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
return 0 # true
else
return 1 # false
fi
}
job=${1}
if ! is_valid "${job}"; then
usage
fi
ContainerName="md_recorder.${job}"
if is_container_running "$ContainerName"; then
echo "Container ${ContainerName} is already running. Aborted."
exit
fi
Cmd="docker run"
Cmd="${Cmd} -d"
Cmd="${Cmd} --rm"
Cmd="${Cmd} --network=host"
Cmd="${Cmd} --pull=always"
Cmd="${Cmd} --name=${ContainerName}"
Cmd="${Cmd} -v /home/cvtt/prod/data:/app/data"
Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
Cmd="${Cmd} cloud21.cvtt.vpn:5500/md_recorder:latest"
Cmd="${Cmd} ${job}"
echo ${Cmd}
eval ${Cmd}
docker/market_data/recorder/md_recorder_monitor.sh
Executable
+61
View File
@@ -0,0 +1,61 @@
#!/bin/bash
usage() {
echo "Usage: $0 <instance (CLD21,CVTTDATA)> [<admin_port (def. 7225)>]"
exit 1
}
ValidInstances=('CLD21' 'CVTTDATA')
is_valid() {
local inst=$1
for valid_inst in "${ValidInstances[@]}";
do
if [[ "$inst" == "$valid_inst" ]]; then
return 0
fi
done
return 1
}
is_container_running() {
local container_name=$1
if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
return 0 # true
else
return 1 # false
fi
}
Instance=${1}
if ! is_valid "${Instance}"; then
usage
fi
AdminPort=7225
if [ "${2}" != "" ]; then
AdminPort=${2}
fi
ContainerName="md_recorder_monitor.${Instance}"
if is_container_running "$ContainerName"; then
echo "Container ${ContainerName} is already running. Aborted."
exit
fi
Cmd="docker run"
Cmd="${Cmd} -d"
Cmd="${Cmd} --rm"
Cmd="${Cmd} --network=host"
Cmd="${Cmd} --pull=always"
Cmd="${Cmd} --name=${ContainerName}"
Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
Cmd="${Cmd} cloud21.cvtt.vpn:5500/md_recorder_monitor:latest"
Cmd="${Cmd} ${Instance} ${AdminPort}"
echo ${Cmd}
eval ${Cmd}