archive/docker_dev
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

progress

Browse Source
This commit is contained in:
Oleg Sheynin 2024-09-10 16:54:38 -04:00
parent 31a1c36939
commit 4e6c9cb349
17 changed files with 307 additions and 254 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ReadMe.md
View File

@ -1,4 +1,4 @@
### Build all docker images from the file **images.txt**
```bash
for dimg in $(cat ./docker_dev/images.txt); do echo $dimg; ./docker_dev/build.sh $dimg; done
for dimg in $(cat ./docker_dev/images.txt | grep -v '#'); do echo $dimg; ./docker_dev/build.sh $dimg; done
```

21
images.txt
View File

@ -1,13 +1,16 @@
config_service
market_data/alpaca_md_day
market_data/crypto_md_day
# Run it like this from cvtt2:
# for dimg in $(cat ./docker_dev/images.txt | grep -v '#'); do echo $dimg; ./docker_dev/build.sh $dimg; done
#
#config_service
#market_data/alpaca_md_day
#market_data/crypto_md_day
market_data/md_portal
market_data/md_recorder
market_data/md_recorder_monitor
market_data/relative_liquidity
research/fastai_jupyter
research/jupyter_gpu_pytorch
research/jupyter_gpu_tensorflow
#market_data/md_recorder
#market_data/md_recorder_monitor
#market_data/relative_liquidity
#research/fastai_jupyter
#research/jupyter_gpu_pytorch
#research/jupyter_gpu_tensorflow
trading/executor
trading/quant
trading/risk_mgr

66
market_data/md_portal/run_md_portal.sh
View File

@ -1,66 +0,0 @@
#!/bin/bash
# runs in container
# runs on host to start container
usage() {
echo -n "Usage: ${0}"
echo -n " [-C <config (dflt: apps/cvtt_eqt_alpaca)>]"
echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
echo -n " [-e <active_exchanges (ALPACA_SNBOX)>]"
echo -n " [-a <admin_port (7220)>]"
echo -n " [-n <portal_name (MD_PORTAL_ALPACA)>]"
echo
exit 1
}
# ConfigServer=cloud16.cvtt.vpn
ConfigServer=cloud23.cvtt.vpn:6789
Config=apps/cvtt_eqt_alpaca
ActiveExchanges=ALPACA_SNDBX
PortalName=MD_PORTAL_ALPACA_SNDBX
AdminPort=7220
while getopts ":C:c:e:a:n:" opt; do
case ${opt} in
C )
Config=$OPTARG
;;
c )
ConfigServer=$OPTARG
;;
e )
ActiveExchanges=$OPTARG
;;
a )
AdminPort=$OPTARG
;;
n )
PortalName=$OPTARG
;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
Cmd="python3.12"
Cmd="${Cmd} cvttpy/apps/md/md_portal.py"
Cmd="${Cmd} --config=http://${ConfigServer}/${Config}"
Cmd="${Cmd} --credentials_file=/.creds"
Cmd="${Cmd} --active_exchanges=${ActiveExchanges}"
Cmd="${Cmd} --portal_name=${PortalName}"
Cmd="${Cmd} --admin_port=${AdminPort}"
Cmd="${Cmd} --log_file=/logs/%T.${PortalName}.log"
Cmd="${Cmd} --compress_log"
echo ${Cmd}
eval ${Cmd}

61
trading/executor/run_executor.sh
View File

@ -5,51 +5,48 @@
# runs on host to start container
usage() {
echo -n "Usage: $0"
echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
echo -n " [-a <admin_port (7222)>"]
echo
exit 1
echo -n "Usage: $0"
echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
echo -n " [-a <admin_port (7222)>"]
echo -n " [-z (compress log)"]
echo
exit 1
}
# ConfigServer=cloud16.cvtt.vpn
ConfigServer=cloud23.cvtt.vpn:6789
Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
ActiveExchanges=ALPACA_SNDBX
AdminPort=7222
COMPRESS_LOG=false
while getopts ":c:e:a:" opt; do
case ${opt} in
c )
ConfigServer=$OPTARG
;;
e )
ActiveExchanges=$OPTARG
;;
a )
AdminPort=$OPTARG
;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
while getopts "c:e:a:z" opt; do
case ${opt} in
c ) Config=$OPTARG ;;
e ) ActiveExchanges=$OPTARG ;;
a ) AdminPort=$OPTARG ;;
z ) COMPRESS_LOG=true ;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
Cmd="python3.12"
Cmd="${Cmd} cvttpy/apps/executor_app.py"
Cmd="${Cmd} --config=http://${ConfigServer}/apps/cvtt_eqt_alpaca"
Cmd="${Cmd} /cvttpy/apps/executor_app.py"
Cmd="${Cmd} --config=${Config}"
Cmd="${Cmd} --credentials_file=/.creds"
Cmd="${Cmd} --allow_dynamic_exch_inst"
Cmd="${Cmd} --active_exchanges=${ActiveExchanges}"
Cmd="${Cmd} --admin_port=${AdminPort}"
Cmd="${Cmd} --log_file=/logs/%T.EXECUTOR_APP.log"
# Cmd="${Cmd} --compress_log"
if ${COMPRESS_LOG}; then
Cmd="${Cmd} --compress_log"
fi
echo ${Cmd}
eval ${Cmd}

0
market_data/md_portal/.creds → trading/md_portal/.creds
View File

0
market_data/md_portal/.dockerignore → trading/md_portal/.dockerignore
View File

2
market_data/md_portal/Dockerfile → trading/md_portal/Dockerfile
View File

@ -1,6 +1,6 @@
FROM python:3.12-slim
ARG FROM_DIR=docker_dev/market_data/md_portal
ARG FROM_DIR=docker_dev/trading/md_portal
COPY ${FROM_DIR}/requirements.txt /
RUN pip install --upgrade pip --root-user-action=ignore

0
market_data/md_portal/requirements.txt → trading/md_portal/requirements.txt
View File

56
trading/md_portal/run_md_portal.sh Executable file
View File

@ -0,0 +1,56 @@
#!/bin/bash
# runs in container
# runs on host to start container
usage() {
echo -n "Usage: ${0}"
echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
echo -n " [-e <active_exchanges (ALPACA_SNBOX)>]"
echo -n " [-a <admin_port (7220)>]"
echo -n " [-n <portal_name (MD_PORTAL_ALPACA)>]"
echo -n " [-z (compress log)"]
echo
exit 1
}
Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
ActiveExchanges=ALPACA_SNDBX
PortalName=MD_PORTAL_ALPACA_SNDBX
AdminPort=7220
COMPRESS_LOG=false
while getopts "c:e:a:n:z" opt; do
case ${opt} in
c ) Config=$OPTARG ;;
e ) ActiveExchanges=$OPTARG ;;
a ) AdminPort=$OPTARG ;;
z ) COMPRESS_LOG=true ;;
n ) PortalName=$OPTARG ;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
Cmd="python3.12"
Cmd="${Cmd} cvttpy/apps/md/md_portal.py"
Cmd="${Cmd} --config=${Config}"
Cmd="${Cmd} --credentials_file=/.creds"
Cmd="${Cmd} --active_exchanges=${ActiveExchanges}"
Cmd="${Cmd} --portal_name=${PortalName}"
Cmd="${Cmd} --admin_port=${AdminPort}"
Cmd="${Cmd} --log_file=/logs/%T.${PortalName}.log"
if ${COMPRESS_LOG}; then
Cmd="${Cmd} --compress_log"
fi
echo ${Cmd}
eval ${Cmd}

69
trading/quant/run_quant.sh
View File

@ -2,51 +2,44 @@
# runs on host to start container
usage() {
echo -n "Usage: $0"
echo -n " -b <book>"
echo -n "Usage: $0"
echo -n " -b <book>"
echo -n " -S <strategy> (dflt: TRDALGO_001)"
echo -n " -S <strategy> (dflt: TRDALGO_001)"
echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
echo -n " [-a <admin_port (7224)>"]
echo
exit 1
echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
echo -n " [-a <admin_port (7224)>"]
echo -n " [-z (compress log)"]
echo
exit 1
}
ConfigServer=cloud23.cvtt.vpn:6789
Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
ActiveExchanges=ALPACA_SNDBX-MDPORTAL
AdminPort=7224
Strategy=DAILY_STOCK_001
Book=""
COMPRESS_LOG=false
while getopts ":b:c:e:a:S:" opt; do
case ${opt} in
c )
ConfigServer=$OPTARG
;;
e )
ActiveExchanges=$OPTARG
;;
a )
AdminPort=$OPTARG
;;
b )
Book=$OPTARG
;;
S )
Strategy=$OPTARG
;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
while getopts "b:c:e:a:S:z" opt; do
case ${opt} in
c ) Config=$OPTARG ;;
e ) ActiveExchanges=$OPTARG ;;
a ) AdminPort=$OPTARG ;;
z ) COMPRESS_LOG=true ;;
b ) Book=$OPTARG ;;
S ) Strategy=$OPTARG ;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
if [ "${Book}" == "" ]; then
@ -56,7 +49,7 @@ fi
Cmd="python3.12"
Cmd="${Cmd} cvttpy/apps/quant_app.py"
Cmd="${Cmd} --config=http://${ConfigServer}/apps/cvtt_eqt_alpaca"
Cmd="${Cmd} --config=${Config}"
Cmd="${Cmd} --credentials_file=/.creds"
Cmd="${Cmd} --book=${Book}"
Cmd="${Cmd} --strategy=${Strategy}"
@ -64,7 +57,9 @@ Cmd="${Cmd} --allow_dynamic_exch_inst"
Cmd="${Cmd} --active_exchanges=${ActiveExchanges}"
Cmd="${Cmd} --admin_port=${AdminPort}"
Cmd="${Cmd} --log_file=/logs/%T.QUANT.${Book}.log"
Cmd="${Cmd} --compress_log"
if ${COMPRESS_LOG}; then
Cmd="${Cmd} --compress_log"
fi
echo ${Cmd}
eval ${Cmd}

64
trading/risk_mgr/run_risk_mgr.sh
View File

@ -4,47 +4,41 @@
# runs on host to start container
usage() {
echo -n "Usage: $0"
echo -n " -b <book>"
echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
echo -n " [-a <admin_port (7223)>"]
echo
exit 1
echo -n "Usage: $0"
echo -n " -b <book>"
echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
echo -n " [-a <admin_port (7223)>"]
echo -n " [-z (compress log)"]
echo
exit 1
}
# ConfigServer=cloud16.cvtt.vpn
ConfigServer=cloud23.cvtt.vpn:6789
Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
ActiveExchanges=ALPACA_SNDBX-MDPORTAL
AdminPort=7223
Book=""
COMPRESS_LOG=false
while getopts ":b:c:e:a:" opt; do
case ${opt} in
c )
ConfigServer=$OPTARG
;;
e )
ActiveExchanges=$OPTARG
;;
a )
AdminPort=$OPTARG
;;
b )
Book=$OPTARG
;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
while getopts "b:c:e:a:z" opt; do
case ${opt} in
c ) Config=$OPTARG ;;
e ) ActiveExchanges=$OPTARG ;;
a ) AdminPort=$OPTARG ;;
z ) COMPRESS_LOG=true ;;
b ) Book=$OPTARG ;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
if [ "${Book}" == "" ]; then
@ -54,14 +48,16 @@ fi
Cmd="python3.12"
Cmd="${Cmd} cvttpy/apps/risk_manager_app.py"
Cmd="${Cmd} --config=http://${ConfigServer}/apps/cvtt_eqt_alpaca"
Cmd="${Cmd} --config=${Config}"
Cmd="${Cmd} --credentials_file=/.creds"
Cmd="${Cmd} --book=${Book}"
Cmd="${Cmd} --allow_dynamic_exch_inst"
Cmd="${Cmd} --active_exchanges=${ActiveExchanges}"
Cmd="${Cmd} --admin_port=${AdminPort}"
Cmd="${Cmd} --log_file=/logs/%T.RISK_MGR.${Book}.log"
Cmd="${Cmd} --compress_log"
if ${COMPRESS_LOG}; then
Cmd="${Cmd} --compress_log"
fi
echo ${Cmd}
eval ${Cmd}

8
trading/service_checker/.creds Normal file
View File

@ -0,0 +1,8 @@
{
"__dummy__": null
, "MATTERMOST": {
"url": "https://mattermost.cryptovaltrading.com"
, "team": "CVTT"
, "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
}
}

26
trading/service_checker/Dockerfile Normal file
View File

@ -0,0 +1,26 @@
FROM python:3.12-slim
ARG FROM_DIR=docker_dev/trading/service_checker
RUN apt-get update && apt-get install -y telnet redis-tools
COPY ${FROM_DIR}/requirements.txt /
RUN pip install --upgrade pip --root-user-action=ignore
RUN pip install -r /requirements.txt --root-user-action=ignore
COPY cvttpy /cvttpy
COPY ${FROM_DIR}/.creds /.creds
COPY ${FROM_DIR}/run_svc_checker.sh /run_svc_checker.sh
RUN chmod +x /run_svc_checker.sh
# RUN mkdir /logs
WORKDIR /
SHELL ["/bin/bash", "-c"]
ENV PYTHONPATH=/
ENTRYPOINT [ "/run_svc_checker.sh" ]
# ENTRYPOINT [ "/usr/bin/bash" ]

7
trading/service_checker/requirements.txt Normal file
View File

@ -0,0 +1,7 @@
aiohttp>=3.7.4.post0
nest-asyncio>=1.5.5
hjson>=3.1.0
sortedcontainers>=2.4.0
redis>=5.0.8
python-dateutil>=2.8.2
types-python-dateutil>=2.8.19.6

41
trading/service_checker/run_svc_checker.sh Executable file
View File

@ -0,0 +1,41 @@
#!/bin/bash
# runs in container
# runs on host to start container
usage() {
echo -n "Usage: ${0}"
echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
echo
exit 1
}
Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
while getopts ":c:" opt; do
case ${opt} in
c )
Config=$OPTARG
;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
Cmd="python3.12"
Cmd="${Cmd} cvttpy/apps/utils/services_checker.py"
Cmd="${Cmd} --config=${Config}"
Cmd="${Cmd} --credentials_file=/.creds"
Cmd="${Cmd} --log_level=ERROR"
echo ${Cmd}
eval ${Cmd}

70
trading/trader/run_trader.sh
View File

@ -5,54 +5,46 @@
# runs on host to start container
usage() {
echo -n "Usage: $0"
echo -n " -b <book>"
echo -n "Usage: $0"
echo -n " -b <book>"
echo -n " -A <algo> (dflt: TRDALGO_001)"
echo -n " -A <algo> (dflt: TRDALGO_001)"
echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
echo -n " [-a <admin_port (7223)>"]
echo
exit 1
echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
echo -n " [-a <admin_port (7223)>"]
echo -n " [-z (compress log)"]
echo
exit 1
}
# ConfigServer=cloud16.cvtt.vpn
ConfigServer=cloud23.cvtt.vpn:6789
Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
ActiveExchanges=ALPACA_SNDBX-MDPORTAL
AdminPort=7226
Algo=TRDALGO_001
Book=""
COMPRESS_LOG=false
while getopts ":b:c:e:a:A:" opt; do
case ${opt} in
c )
ConfigServer=$OPTARG
;;
e )
ActiveExchanges=$OPTARG
;;
a )
AdminPort=$OPTARG
;;
b )
Book=$OPTARG
;;
A )
Algo=$OPTARG
;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
while getopts "b:c:e:a:A:z" opt; do
case ${opt} in
c ) Config=$OPTARG ;;
e ) ActiveExchanges=$OPTARG ;;
a ) AdminPort=$OPTARG ;;
z ) COMPRESS_LOG=true ;;
b ) Book=$OPTARG ;;
A ) Algo=$OPTARG ;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
if [ "${Book}" == "" ]; then
@ -62,7 +54,7 @@ fi
Cmd="python3.12"
Cmd="${Cmd} cvttpy/apps/trader_app.py"
Cmd="${Cmd} --config=http://${ConfigServer}/apps/cvtt_eqt_alpaca"
Cmd="${Cmd} --config=${Config}"
Cmd="${Cmd} --credentials_file=/.creds"
Cmd="${Cmd} --book=${Book}"
Cmd="${Cmd} --algo=${Algo}"
@ -70,7 +62,9 @@ Cmd="${Cmd} --allow_dynamic_exch_inst"
Cmd="${Cmd} --active_exchanges=${ActiveExchanges}"
Cmd="${Cmd} --admin_port=${AdminPort}"
Cmd="${Cmd} --log_file=/logs/%T.TRADER.${Book}.log"
Cmd="${Cmd} --compress_log"
if ${COMPRESS_LOG}; then
Cmd="${Cmd} --compress_log"
fi
echo ${Cmd}
eval ${Cmd}

68
trading/trading_recorder/run_trading_recorder.sh
View File

@ -4,46 +4,40 @@
# runs on host to start container
usage() {
echo -n "Usage: $0"
echo -n " -b <book>"
echo -n " [-C <config (dflt: apps/cvtt_eqt_alpaca)>]"
echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
echo -n " [-a <admin_port (7223)>"]
echo
exit 1
echo -n "Usage: $0"
echo -n " -b <book>"
echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
echo -n " [-a <admin_port (7223)>"]
echo -n " [-z (compress log)"]
echo
exit 1
}
# ConfigServer=cloud16.cvtt.vpn
ConfigServer=cloud23.cvtt.vpn:6789
ActiveExchanges=ALPACA_SNDBX-MDPORTAL
AdminPort=7225
Book=""
Config=apps/cvtt_eqt_alpaca
Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
COMPRESS_LOG=false
while getopts ":C:b:c:e:a:" opt; do
case ${opt} in
c )
ConfigServer=$OPTARG
;;
e )
ActiveExchanges=$OPTARG
;;
a )
AdminPort=$OPTARG
;;
b )
Book=$OPTARG
;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
while getopts "b:c:e:a:z" opt; do
case ${opt} in
c ) Config=$OPTARG ;;
e ) ActiveExchanges=$OPTARG ;;
a ) AdminPort=$OPTARG ;;
z ) COMPRESS_LOG=true ;;
b ) Book=$OPTARG ;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
if [ "${Book}" == "" ]; then
@ -52,14 +46,16 @@ if [ "${Book}" == "" ]; then
fi
Cmd="python3.12"
Cmd="${Cmd} cvttpy/apps/trader_app.py"
Cmd="${Cmd} --config=http://${ConfigServer}/${Config}"
Cmd="${Cmd} cvttpy/apps/trade_recorder.py"
Cmd="${Cmd} --config=${Config}"
Cmd="${Cmd} --credentials_file=/.creds"
Cmd="${Cmd} --book=${Book}"
Cmd="${Cmd} --allow_dynamic_exch_inst"
Cmd="${Cmd} --active_exchanges=${ActiveExchanges}"
Cmd="${Cmd} --admin_port=${AdminPort}"
Cmd="${Cmd} --log_file=/logs/%T.TRADING_RECORDER.${Book}.log"
Cmd="${Cmd} --compress_log"
if ${COMPRESS_LOG}; then
Cmd="${Cmd} --compress_log"
fi
echo ${Cmd}
eval ${Cmd}