diff --git a/ReadMe.md b/ReadMe.md index 269aa53..55914d5 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -1,4 +1,4 @@ ### Build all docker images from the file **images.txt** ```bash -for dimg in $(cat ./docker_dev/images.txt); do echo $dimg; ./docker_dev/build.sh $dimg; done +for dimg in $(cat ./docker_dev/images.txt | grep -v '#'); do echo $dimg; ./docker_dev/build.sh $dimg; done ``` \ No newline at end of file diff --git a/images.txt b/images.txt index 167acc7..91634a7 100644 --- a/images.txt +++ b/images.txt @@ -1,13 +1,16 @@ -config_service -market_data/alpaca_md_day -market_data/crypto_md_day +# Run it like this from cvtt2: +# for dimg in $(cat ./docker_dev/images.txt | grep -v '#'); do echo $dimg; ./docker_dev/build.sh $dimg; done +# +#config_service +#market_data/alpaca_md_day +#market_data/crypto_md_day market_data/md_portal -market_data/md_recorder -market_data/md_recorder_monitor -market_data/relative_liquidity -research/fastai_jupyter -research/jupyter_gpu_pytorch -research/jupyter_gpu_tensorflow +#market_data/md_recorder +#market_data/md_recorder_monitor +#market_data/relative_liquidity +#research/fastai_jupyter +#research/jupyter_gpu_pytorch +#research/jupyter_gpu_tensorflow trading/executor trading/quant trading/risk_mgr diff --git a/market_data/md_portal/run_md_portal.sh b/market_data/md_portal/run_md_portal.sh deleted file mode 100755 index d1ea540..0000000 --- a/market_data/md_portal/run_md_portal.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -# runs in container - -# runs on host to start container -usage() { - echo -n "Usage: ${0}" - echo -n " [-C ]" - echo -n " [-c ]" - echo -n " [-e ]" - echo -n " [-a ]" - echo -n " [-n ]" - echo - exit 1 -} - -# ConfigServer=cloud16.cvtt.vpn -ConfigServer=cloud23.cvtt.vpn:6789 -Config=apps/cvtt_eqt_alpaca -ActiveExchanges=ALPACA_SNDBX -PortalName=MD_PORTAL_ALPACA_SNDBX -AdminPort=7220 - - -while getopts ":C:c:e:a:n:" opt; do - case ${opt} in - C ) - Config=$OPTARG - ;; - c ) - ConfigServer=$OPTARG - ;; - e ) - ActiveExchanges=$OPTARG - ;; - a ) - AdminPort=$OPTARG - ;; - n ) - PortalName=$OPTARG - ;; - \? ) - echo "Invalid option: -$OPTARG" >&2 - usage - ;; - : ) - echo "Option -$OPTARG requires an argument." >&2 - usage - ;; - esac -done - - -Cmd="python3.12" -Cmd="${Cmd} cvttpy/apps/md/md_portal.py" -Cmd="${Cmd} --config=http://${ConfigServer}/${Config}" -Cmd="${Cmd} --credentials_file=/.creds" -Cmd="${Cmd} --active_exchanges=${ActiveExchanges}" -Cmd="${Cmd} --portal_name=${PortalName}" -Cmd="${Cmd} --admin_port=${AdminPort}" -Cmd="${Cmd} --log_file=/logs/%T.${PortalName}.log" -Cmd="${Cmd} --compress_log" -echo ${Cmd} -eval ${Cmd} - - diff --git a/trading/executor/run_executor.sh b/trading/executor/run_executor.sh index 8fa9127..86dd1bd 100755 --- a/trading/executor/run_executor.sh +++ b/trading/executor/run_executor.sh @@ -5,51 +5,48 @@ # runs on host to start container usage() { - echo -n "Usage: $0" - echo -n " [-c ]" - echo -n " [-e ]" - echo -n " [-a "] - echo - exit 1 + echo -n "Usage: $0" + echo -n " [-c ]" + echo -n " [-e ]" + echo -n " [-a "] + echo -n " [-z (compress log)"] + echo + exit 1 } -# ConfigServer=cloud16.cvtt.vpn -ConfigServer=cloud23.cvtt.vpn:6789 +Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca ActiveExchanges=ALPACA_SNDBX AdminPort=7222 +COMPRESS_LOG=false - -while getopts ":c:e:a:" opt; do - case ${opt} in - c ) - ConfigServer=$OPTARG - ;; - e ) - ActiveExchanges=$OPTARG - ;; - a ) - AdminPort=$OPTARG - ;; - \? ) - echo "Invalid option: -$OPTARG" >&2 - usage - ;; - : ) - echo "Option -$OPTARG requires an argument." >&2 - usage - ;; - esac +while getopts "c:e:a:z" opt; do + case ${opt} in + c ) Config=$OPTARG ;; + e ) ActiveExchanges=$OPTARG ;; + a ) AdminPort=$OPTARG ;; + z ) COMPRESS_LOG=true ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + usage + ;; + : ) + echo "Option -$OPTARG requires an argument." >&2 + usage + ;; + esac done Cmd="python3.12" -Cmd="${Cmd} cvttpy/apps/executor_app.py" -Cmd="${Cmd} --config=http://${ConfigServer}/apps/cvtt_eqt_alpaca" +Cmd="${Cmd} /cvttpy/apps/executor_app.py" +Cmd="${Cmd} --config=${Config}" Cmd="${Cmd} --credentials_file=/.creds" Cmd="${Cmd} --allow_dynamic_exch_inst" Cmd="${Cmd} --active_exchanges=${ActiveExchanges}" Cmd="${Cmd} --admin_port=${AdminPort}" Cmd="${Cmd} --log_file=/logs/%T.EXECUTOR_APP.log" -# Cmd="${Cmd} --compress_log" +if ${COMPRESS_LOG}; then + Cmd="${Cmd} --compress_log" +fi echo ${Cmd} eval ${Cmd} diff --git a/market_data/md_portal/.creds b/trading/md_portal/.creds similarity index 100% rename from market_data/md_portal/.creds rename to trading/md_portal/.creds diff --git a/market_data/md_portal/.dockerignore b/trading/md_portal/.dockerignore similarity index 100% rename from market_data/md_portal/.dockerignore rename to trading/md_portal/.dockerignore diff --git a/market_data/md_portal/Dockerfile b/trading/md_portal/Dockerfile similarity index 90% rename from market_data/md_portal/Dockerfile rename to trading/md_portal/Dockerfile index 89af514..4405008 100644 --- a/market_data/md_portal/Dockerfile +++ b/trading/md_portal/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.12-slim -ARG FROM_DIR=docker_dev/market_data/md_portal +ARG FROM_DIR=docker_dev/trading/md_portal COPY ${FROM_DIR}/requirements.txt / RUN pip install --upgrade pip --root-user-action=ignore diff --git a/market_data/md_portal/requirements.txt b/trading/md_portal/requirements.txt similarity index 100% rename from market_data/md_portal/requirements.txt rename to trading/md_portal/requirements.txt diff --git a/trading/md_portal/run_md_portal.sh b/trading/md_portal/run_md_portal.sh new file mode 100755 index 0000000..68c8b62 --- /dev/null +++ b/trading/md_portal/run_md_portal.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# runs in container + +# runs on host to start container +usage() { + echo -n "Usage: ${0}" + echo -n " [-c ]" + echo -n " [-e ]" + echo -n " [-a ]" + echo -n " [-n ]" + echo -n " [-z (compress log)"] + echo + exit 1 +} + +Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca +ActiveExchanges=ALPACA_SNDBX +PortalName=MD_PORTAL_ALPACA_SNDBX +AdminPort=7220 +COMPRESS_LOG=false + + +while getopts "c:e:a:n:z" opt; do + case ${opt} in + c ) Config=$OPTARG ;; + e ) ActiveExchanges=$OPTARG ;; + a ) AdminPort=$OPTARG ;; + z ) COMPRESS_LOG=true ;; + n ) PortalName=$OPTARG ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + usage + ;; + : ) + echo "Option -$OPTARG requires an argument." >&2 + usage + ;; + esac +done + +Cmd="python3.12" +Cmd="${Cmd} cvttpy/apps/md/md_portal.py" +Cmd="${Cmd} --config=${Config}" +Cmd="${Cmd} --credentials_file=/.creds" +Cmd="${Cmd} --active_exchanges=${ActiveExchanges}" +Cmd="${Cmd} --portal_name=${PortalName}" +Cmd="${Cmd} --admin_port=${AdminPort}" +Cmd="${Cmd} --log_file=/logs/%T.${PortalName}.log" +if ${COMPRESS_LOG}; then + Cmd="${Cmd} --compress_log" +fi +echo ${Cmd} +eval ${Cmd} + + diff --git a/trading/quant/run_quant.sh b/trading/quant/run_quant.sh index 63a43e6..0e812a6 100755 --- a/trading/quant/run_quant.sh +++ b/trading/quant/run_quant.sh @@ -2,51 +2,44 @@ # runs on host to start container usage() { - echo -n "Usage: $0" - echo -n " -b " + echo -n "Usage: $0" + echo -n " -b " - echo -n " -S (dflt: TRDALGO_001)" + echo -n " -S (dflt: TRDALGO_001)" - echo -n " [-c ]" - echo -n " [-e ]" - echo -n " [-a "] - echo - exit 1 + echo -n " [-c ]" + echo -n " [-e ]" + echo -n " [-a "] + echo -n " [-z (compress log)"] + echo + exit 1 } -ConfigServer=cloud23.cvtt.vpn:6789 +Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca ActiveExchanges=ALPACA_SNDBX-MDPORTAL AdminPort=7224 Strategy=DAILY_STOCK_001 Book="" +COMPRESS_LOG=false -while getopts ":b:c:e:a:S:" opt; do - case ${opt} in - c ) - ConfigServer=$OPTARG - ;; - e ) - ActiveExchanges=$OPTARG - ;; - a ) - AdminPort=$OPTARG - ;; - b ) - Book=$OPTARG - ;; - S ) - Strategy=$OPTARG - ;; - \? ) - echo "Invalid option: -$OPTARG" >&2 - usage - ;; - : ) - echo "Option -$OPTARG requires an argument." >&2 - usage - ;; - esac +while getopts "b:c:e:a:S:z" opt; do + case ${opt} in + c ) Config=$OPTARG ;; + e ) ActiveExchanges=$OPTARG ;; + a ) AdminPort=$OPTARG ;; + z ) COMPRESS_LOG=true ;; + b ) Book=$OPTARG ;; + S ) Strategy=$OPTARG ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + usage + ;; + : ) + echo "Option -$OPTARG requires an argument." >&2 + usage + ;; + esac done if [ "${Book}" == "" ]; then @@ -56,7 +49,7 @@ fi Cmd="python3.12" Cmd="${Cmd} cvttpy/apps/quant_app.py" -Cmd="${Cmd} --config=http://${ConfigServer}/apps/cvtt_eqt_alpaca" +Cmd="${Cmd} --config=${Config}" Cmd="${Cmd} --credentials_file=/.creds" Cmd="${Cmd} --book=${Book}" Cmd="${Cmd} --strategy=${Strategy}" @@ -64,7 +57,9 @@ Cmd="${Cmd} --allow_dynamic_exch_inst" Cmd="${Cmd} --active_exchanges=${ActiveExchanges}" Cmd="${Cmd} --admin_port=${AdminPort}" Cmd="${Cmd} --log_file=/logs/%T.QUANT.${Book}.log" -Cmd="${Cmd} --compress_log" +if ${COMPRESS_LOG}; then + Cmd="${Cmd} --compress_log" +fi echo ${Cmd} eval ${Cmd} diff --git a/trading/risk_mgr/run_risk_mgr.sh b/trading/risk_mgr/run_risk_mgr.sh index 4c285ad..61f776f 100755 --- a/trading/risk_mgr/run_risk_mgr.sh +++ b/trading/risk_mgr/run_risk_mgr.sh @@ -4,47 +4,41 @@ # runs on host to start container usage() { - echo -n "Usage: $0" - echo -n " -b " - echo -n " [-c ]" - echo -n " [-e ]" - echo -n " [-a "] - echo - exit 1 + echo -n "Usage: $0" + echo -n " -b " + echo -n " [-c ]" + echo -n " [-e ]" + echo -n " [-a "] + echo -n " [-z (compress log)"] + echo + exit 1 } -# ConfigServer=cloud16.cvtt.vpn -ConfigServer=cloud23.cvtt.vpn:6789 +Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca ActiveExchanges=ALPACA_SNDBX-MDPORTAL AdminPort=7223 Book="" +COMPRESS_LOG=false -while getopts ":b:c:e:a:" opt; do - case ${opt} in - c ) - ConfigServer=$OPTARG - ;; - e ) - ActiveExchanges=$OPTARG - ;; - a ) - AdminPort=$OPTARG - ;; - b ) - Book=$OPTARG - ;; - \? ) - echo "Invalid option: -$OPTARG" >&2 - usage - ;; - : ) - echo "Option -$OPTARG requires an argument." >&2 - usage - ;; - esac +while getopts "b:c:e:a:z" opt; do + case ${opt} in + c ) Config=$OPTARG ;; + e ) ActiveExchanges=$OPTARG ;; + a ) AdminPort=$OPTARG ;; + z ) COMPRESS_LOG=true ;; + b ) Book=$OPTARG ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + usage + ;; + : ) + echo "Option -$OPTARG requires an argument." >&2 + usage + ;; + esac done if [ "${Book}" == "" ]; then @@ -54,14 +48,16 @@ fi Cmd="python3.12" Cmd="${Cmd} cvttpy/apps/risk_manager_app.py" -Cmd="${Cmd} --config=http://${ConfigServer}/apps/cvtt_eqt_alpaca" +Cmd="${Cmd} --config=${Config}" Cmd="${Cmd} --credentials_file=/.creds" Cmd="${Cmd} --book=${Book}" Cmd="${Cmd} --allow_dynamic_exch_inst" Cmd="${Cmd} --active_exchanges=${ActiveExchanges}" Cmd="${Cmd} --admin_port=${AdminPort}" Cmd="${Cmd} --log_file=/logs/%T.RISK_MGR.${Book}.log" -Cmd="${Cmd} --compress_log" +if ${COMPRESS_LOG}; then + Cmd="${Cmd} --compress_log" +fi echo ${Cmd} eval ${Cmd} diff --git a/trading/service_checker/.creds b/trading/service_checker/.creds new file mode 100644 index 0000000..aae4c4d --- /dev/null +++ b/trading/service_checker/.creds @@ -0,0 +1,8 @@ +{ + "__dummy__": null + , "MATTERMOST": { + "url": "https://mattermost.cryptovaltrading.com" + , "team": "CVTT" + , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano" + } +} diff --git a/trading/service_checker/Dockerfile b/trading/service_checker/Dockerfile new file mode 100644 index 0000000..e6850df --- /dev/null +++ b/trading/service_checker/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.12-slim + +ARG FROM_DIR=docker_dev/trading/service_checker + +RUN apt-get update && apt-get install -y telnet redis-tools + + +COPY ${FROM_DIR}/requirements.txt / +RUN pip install --upgrade pip --root-user-action=ignore +RUN pip install -r /requirements.txt --root-user-action=ignore + +COPY cvttpy /cvttpy +COPY ${FROM_DIR}/.creds /.creds + + +COPY ${FROM_DIR}/run_svc_checker.sh /run_svc_checker.sh +RUN chmod +x /run_svc_checker.sh + +# RUN mkdir /logs + +WORKDIR / +SHELL ["/bin/bash", "-c"] +ENV PYTHONPATH=/ +ENTRYPOINT [ "/run_svc_checker.sh" ] +# ENTRYPOINT [ "/usr/bin/bash" ] + diff --git a/trading/service_checker/requirements.txt b/trading/service_checker/requirements.txt new file mode 100644 index 0000000..852e647 --- /dev/null +++ b/trading/service_checker/requirements.txt @@ -0,0 +1,7 @@ +aiohttp>=3.7.4.post0 +nest-asyncio>=1.5.5 +hjson>=3.1.0 +sortedcontainers>=2.4.0 +redis>=5.0.8 +python-dateutil>=2.8.2 +types-python-dateutil>=2.8.19.6 diff --git a/trading/service_checker/run_svc_checker.sh b/trading/service_checker/run_svc_checker.sh new file mode 100755 index 0000000..4f5baf5 --- /dev/null +++ b/trading/service_checker/run_svc_checker.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# runs in container + +# runs on host to start container +usage() { + echo -n "Usage: ${0}" + echo -n " [-c ]" + echo + exit 1 +} + +Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca + + +while getopts ":c:" opt; do + case ${opt} in + c ) + Config=$OPTARG + ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + usage + ;; + : ) + echo "Option -$OPTARG requires an argument." >&2 + usage + ;; + esac +done + + +Cmd="python3.12" +Cmd="${Cmd} cvttpy/apps/utils/services_checker.py" +Cmd="${Cmd} --config=${Config}" +Cmd="${Cmd} --credentials_file=/.creds" +Cmd="${Cmd} --log_level=ERROR" + +echo ${Cmd} +eval ${Cmd} + diff --git a/trading/trader/run_trader.sh b/trading/trader/run_trader.sh index 6145c38..8439345 100755 --- a/trading/trader/run_trader.sh +++ b/trading/trader/run_trader.sh @@ -5,54 +5,46 @@ # runs on host to start container usage() { - echo -n "Usage: $0" - echo -n " -b " + echo -n "Usage: $0" + echo -n " -b " - echo -n " -A (dflt: TRDALGO_001)" + echo -n " -A (dflt: TRDALGO_001)" - echo -n " [-c ]" - echo -n " [-e ]" - echo -n " [-a "] - echo - exit 1 + echo -n " [-c ]" + echo -n " [-e ]" + echo -n " [-a "] + echo -n " [-z (compress log)"] + echo + exit 1 } -# ConfigServer=cloud16.cvtt.vpn -ConfigServer=cloud23.cvtt.vpn:6789 +Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca ActiveExchanges=ALPACA_SNDBX-MDPORTAL AdminPort=7226 Algo=TRDALGO_001 Book="" +COMPRESS_LOG=false -while getopts ":b:c:e:a:A:" opt; do - case ${opt} in - c ) - ConfigServer=$OPTARG - ;; - e ) - ActiveExchanges=$OPTARG - ;; - a ) - AdminPort=$OPTARG - ;; - b ) - Book=$OPTARG - ;; - A ) - Algo=$OPTARG - ;; - \? ) - echo "Invalid option: -$OPTARG" >&2 - usage - ;; - : ) - echo "Option -$OPTARG requires an argument." >&2 - usage - ;; - esac +while getopts "b:c:e:a:A:z" opt; do + case ${opt} in + c ) Config=$OPTARG ;; + e ) ActiveExchanges=$OPTARG ;; + a ) AdminPort=$OPTARG ;; + z ) COMPRESS_LOG=true ;; + b ) Book=$OPTARG ;; + A ) Algo=$OPTARG ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + usage + ;; + : ) + echo "Option -$OPTARG requires an argument." >&2 + usage + ;; + esac done if [ "${Book}" == "" ]; then @@ -62,7 +54,7 @@ fi Cmd="python3.12" Cmd="${Cmd} cvttpy/apps/trader_app.py" -Cmd="${Cmd} --config=http://${ConfigServer}/apps/cvtt_eqt_alpaca" +Cmd="${Cmd} --config=${Config}" Cmd="${Cmd} --credentials_file=/.creds" Cmd="${Cmd} --book=${Book}" Cmd="${Cmd} --algo=${Algo}" @@ -70,7 +62,9 @@ Cmd="${Cmd} --allow_dynamic_exch_inst" Cmd="${Cmd} --active_exchanges=${ActiveExchanges}" Cmd="${Cmd} --admin_port=${AdminPort}" Cmd="${Cmd} --log_file=/logs/%T.TRADER.${Book}.log" -Cmd="${Cmd} --compress_log" +if ${COMPRESS_LOG}; then + Cmd="${Cmd} --compress_log" +fi echo ${Cmd} eval ${Cmd} diff --git a/trading/trading_recorder/run_trading_recorder.sh b/trading/trading_recorder/run_trading_recorder.sh index 4b51d3c..ea97ae1 100755 --- a/trading/trading_recorder/run_trading_recorder.sh +++ b/trading/trading_recorder/run_trading_recorder.sh @@ -4,46 +4,40 @@ # runs on host to start container usage() { - echo -n "Usage: $0" - echo -n " -b " - echo -n " [-C ]" - echo -n " [-c ]" - echo -n " [-e ]" - echo -n " [-a "] - echo - exit 1 + echo -n "Usage: $0" + echo -n " -b " + echo -n " [-c ]" + echo -n " [-e ]" + echo -n " [-a "] + echo -n " [-z (compress log)"] + echo + exit 1 } # ConfigServer=cloud16.cvtt.vpn -ConfigServer=cloud23.cvtt.vpn:6789 ActiveExchanges=ALPACA_SNDBX-MDPORTAL AdminPort=7225 Book="" -Config=apps/cvtt_eqt_alpaca +Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca +COMPRESS_LOG=false -while getopts ":C:b:c:e:a:" opt; do - case ${opt} in - c ) - ConfigServer=$OPTARG - ;; - e ) - ActiveExchanges=$OPTARG - ;; - a ) - AdminPort=$OPTARG - ;; - b ) - Book=$OPTARG - ;; - \? ) - echo "Invalid option: -$OPTARG" >&2 - usage - ;; - : ) - echo "Option -$OPTARG requires an argument." >&2 - usage - ;; - esac + +while getopts "b:c:e:a:z" opt; do + case ${opt} in + c ) Config=$OPTARG ;; + e ) ActiveExchanges=$OPTARG ;; + a ) AdminPort=$OPTARG ;; + z ) COMPRESS_LOG=true ;; + b ) Book=$OPTARG ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + usage + ;; + : ) + echo "Option -$OPTARG requires an argument." >&2 + usage + ;; + esac done if [ "${Book}" == "" ]; then @@ -52,14 +46,16 @@ if [ "${Book}" == "" ]; then fi Cmd="python3.12" -Cmd="${Cmd} cvttpy/apps/trader_app.py" -Cmd="${Cmd} --config=http://${ConfigServer}/${Config}" +Cmd="${Cmd} cvttpy/apps/trade_recorder.py" +Cmd="${Cmd} --config=${Config}" Cmd="${Cmd} --credentials_file=/.creds" Cmd="${Cmd} --book=${Book}" Cmd="${Cmd} --allow_dynamic_exch_inst" Cmd="${Cmd} --active_exchanges=${ActiveExchanges}" Cmd="${Cmd} --admin_port=${AdminPort}" Cmd="${Cmd} --log_file=/logs/%T.TRADING_RECORDER.${Book}.log" -Cmd="${Cmd} --compress_log" +if ${COMPRESS_LOG}; then + Cmd="${Cmd} --compress_log" +fi echo ${Cmd} eval ${Cmd}