diff --git a/build.sh b/build.sh index aa34211..5a3917e 100755 --- a/build.sh +++ b/build.sh @@ -1,27 +1,28 @@ #!/bin/bash function usage { - echo "Usage: ${0} " + echo "Usage: ${0} " exit } -AppName=${1} +AppPath=${1} -if [ "${AppName}" == "" ] +if [ "${AppPath}" == "" ] then usage fi # --- Settings -DockerDir=$(realpath $(dirname ${0})/${AppName}) -RootDir=$(realpath ${DockerDir}/../..) +DockerDir=$(realpath $(dirname ${0})/${AppPath}) +RootDir=$(realpath $(dirname ${0})/..) RegistryService=cloud21.cvtt.vpn:5500 +AppName=$(basename ${AppPath}) + ImageName=${AppName} ImageDir=${HOME}/docker_images mkdir -p ${ImageDir} - Version=$(cat ${RootDir}/cvttpy/release_version.txt) cd ${RootDir} diff --git a/alpaca_md_day/.creds b/market_data/alpaca_md_day/.creds similarity index 100% rename from alpaca_md_day/.creds rename to market_data/alpaca_md_day/.creds diff --git a/alpaca_md_day/Dockerfile b/market_data/alpaca_md_day/Dockerfile similarity index 79% rename from alpaca_md_day/Dockerfile rename to market_data/alpaca_md_day/Dockerfile index a8e8f76..c6c5523 100644 --- a/alpaca_md_day/Dockerfile +++ b/market_data/alpaca_md_day/Dockerfile @@ -1,5 +1,7 @@ FROM python:3.10-slim +ARG FROM_DIR=docker_dev/market_data/alpaca_md_day + # Update the package list and install required packages RUN apt-get update && apt-get install -y --no-install-recommends \ apt-utils \ @@ -13,13 +15,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN pip install --upgrade pip -COPY docker_dev/alpaca_md_day/requirements.txt / +COPY ${FROM_DIR}/requirements.txt / RUN pip install -r /requirements.txt COPY cvttpy /cvttpy -COPY docker_dev/alpaca_md_day/.creds /.creds -COPY docker_dev/alpaca_md_day/.creds /root/.creds -COPY docker_dev/alpaca_md_day/alpaca_md_day.sh /alpaca_md_day.sh +COPY ${FROM_DIR}/.creds /.creds +COPY ${FROM_DIR}/.creds /root/.creds +COPY ${FROM_DIR}/alpaca_md_day.sh /alpaca_md_day.sh COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub diff --git a/alpaca_md_day/HOWTO.md b/market_data/alpaca_md_day/HOWTO.md similarity index 100% rename from alpaca_md_day/HOWTO.md rename to market_data/alpaca_md_day/HOWTO.md diff --git a/alpaca_md_day/alpaca_md_day.sh b/market_data/alpaca_md_day/alpaca_md_day.sh similarity index 100% rename from alpaca_md_day/alpaca_md_day.sh rename to market_data/alpaca_md_day/alpaca_md_day.sh diff --git a/alpaca_md_day/requirements.txt b/market_data/alpaca_md_day/requirements.txt similarity index 100% rename from alpaca_md_day/requirements.txt rename to market_data/alpaca_md_day/requirements.txt diff --git a/crypto_md_day/.creds b/market_data/crypto_md_day/.creds similarity index 100% rename from crypto_md_day/.creds rename to market_data/crypto_md_day/.creds diff --git a/crypto_md_day/Dockerfile b/market_data/crypto_md_day/Dockerfile similarity index 80% rename from crypto_md_day/Dockerfile rename to market_data/crypto_md_day/Dockerfile index d8b5dcb..5ae92c4 100644 --- a/crypto_md_day/Dockerfile +++ b/market_data/crypto_md_day/Dockerfile @@ -1,13 +1,14 @@ FROM python:3.10-slim +ARG FROM_DIR=docker_dev/market_data/crypto_md_day -COPY docker_dev/crypto_md_day/requirements.txt / +COPY ${FROM_DIR}/requirements.txt / RUN pip install --upgrade pip --root-user-action=ignore RUN pip install -r /requirements.txt --root-user-action=ignore COPY cvttpy /cvttpy -COPY docker_dev/crypto_md_day/.creds /.creds -COPY docker_dev/crypto_md_day/crypto_md_day.sh /crypto_md_day.sh +COPY ${FROM_DIR}/.creds /.creds +COPY ${FROM_DIR}/crypto_md_day.sh /crypto_md_day.sh RUN apt-get update && apt-get install -y --no-install-recommends rsync openssh-client && rm -rf /var/lib/apt/lists/* diff --git a/crypto_md_day/HOWTO.md b/market_data/crypto_md_day/HOWTO.md similarity index 100% rename from crypto_md_day/HOWTO.md rename to market_data/crypto_md_day/HOWTO.md diff --git a/crypto_md_day/crypto_md_day.sh b/market_data/crypto_md_day/crypto_md_day.sh similarity index 100% rename from crypto_md_day/crypto_md_day.sh rename to market_data/crypto_md_day/crypto_md_day.sh diff --git a/crypto_md_day/requirements.txt b/market_data/crypto_md_day/requirements.txt similarity index 100% rename from crypto_md_day/requirements.txt rename to market_data/crypto_md_day/requirements.txt diff --git a/md_recorder/.creds b/market_data/md_recorder/.creds similarity index 100% rename from md_recorder/.creds rename to market_data/md_recorder/.creds diff --git a/md_recorder/Dockerfile b/market_data/md_recorder/Dockerfile similarity index 66% rename from md_recorder/Dockerfile rename to market_data/md_recorder/Dockerfile index 4cb9151..08c5d9c 100644 --- a/md_recorder/Dockerfile +++ b/market_data/md_recorder/Dockerfile @@ -1,13 +1,15 @@ FROM python:3.10-slim +ARG FROM_DIR=docker_dev/market_data/md_recorder -COPY docker_dev/md_recorder/requirements.txt / + +COPY ${FROM_DIR}/requirements.txt / RUN pip install --upgrade pip --root-user-action=ignore RUN pip install -r /requirements.txt --root-user-action=ignore COPY cvttpy /cvttpy -COPY docker_dev/md_recorder/.creds /.creds -COPY docker_dev/md_recorder/run_md_recorder.sh /run_md_recorder.sh +COPY ${FROM_DIR}/.creds /.creds +COPY ${FROM_DIR}/run_md_recorder.sh /run_md_recorder.sh # Shared Volumes diff --git a/md_recorder/requirements.txt b/market_data/md_recorder/requirements.txt similarity index 100% rename from md_recorder/requirements.txt rename to market_data/md_recorder/requirements.txt diff --git a/md_recorder/run_md_recorder.sh b/market_data/md_recorder/run_md_recorder.sh similarity index 56% rename from md_recorder/run_md_recorder.sh rename to market_data/md_recorder/run_md_recorder.sh index 1ba88c9..82d04f2 100755 --- a/md_recorder/run_md_recorder.sh +++ b/market_data/md_recorder/run_md_recorder.sh @@ -2,24 +2,37 @@ # runs in container +ValidJobs=('BNBFUT_CLD21' 'BNBSPOT_CLD21' 'COINBASE_CLD21' 'BNBFUT_CVTTDATA' 'BNBSPOT_CVTTDATA' 'COINBASE_CVTTDATA') +# runs on host to start container usage() { - echo "Usage: $0 (BNBFUT_CLD21, BNBSPOT_CLD21, BNBSPOT_CVTTDATA, COINBASE_CLD21)" + echo "Usage: $0 " exit 1 } +is_valid() { + local job=$1 + for valid_job in "${ValidJobs[@]}"; + do + # echo "job=$job valid_job=$valid_job" + if [[ "${job}" == "${valid_job}" ]]; then + return 0 + fi + done + return 1 +} job=${1} -if [ "${job}" == "" ] ; then +if ! is_valid "${job}"; then usage fi # ConfigServer=cloud16.cvtt.vpn -ConfigServer=cloud23.cvtt.vpn +ConfigServer=cloud23.cvtt.vpn:6789 +Cmd="python3.10" +Cmd="${Cmd} cvttpy/apps/md/md_recorder.py" +Cmd="${Cmd} --config=http://${ConfigServer}/apps/md_recorder" if [ "${job}" == "BNBFUT_CLD21" ] ; then - Cmd="python3.10" - Cmd="${Cmd} cvttpy/apps/md/md_recorder.py" - Cmd="${Cmd} --config=http://${ConfigServer}:6789/apps/md_recorder" Cmd="${Cmd} --active_exchanges=BNBFUT" Cmd="${Cmd} --instruments=BNBFUT:PERP-BTC-USDT,BNBFUT:PERP-ETH-USDT" Cmd="${Cmd} --credentials_file=/.creds" @@ -27,9 +40,6 @@ if [ "${job}" == "BNBFUT_CLD21" ] ; then Cmd="${Cmd} --admin_port=7201" Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBFUT.CLD21.log" elif [ "${job}" == "COINBASE_CLD21" ] ; then - Cmd="python3.10" - Cmd="${Cmd} cvttpy/apps/md/md_recorder.py" - Cmd="${Cmd} --config=http://${ConfigServer}:6789/apps/md_recorder" Cmd="${Cmd} --active_exchanges=COINBASE_AT" Cmd="${Cmd} --instruments=COINBASE:PAIR-BTC-USD,COINBASE:PAIR-ETH-USD,COINBASE:PAIR-LTC-USD,COINBASE:PAIR-XRP-USD" Cmd="${Cmd} --credentials_file=/.creds" @@ -37,9 +47,6 @@ elif [ "${job}" == "COINBASE_CLD21" ] ; then Cmd="${Cmd} --admin_port=7202" Cmd="${Cmd} --log_file=/logs/%T.MD_REC.COINBASE.CLD21.log" elif [ "${job}" == "BNBSPOT_CLD21" ] ; then - Cmd="python3.10" - Cmd="${Cmd} cvttpy/apps/md/md_recorder.py" - Cmd="${Cmd} --config=http://${ConfigServer}:6789/apps/md_recorder" Cmd="${Cmd} --active_exchanges=BNBSPOT" Cmd="${Cmd} --instruments=BNBSPOT:PAIR-BTC-USDT,BNBSPOT:PAIR-ETH-USDT,BNBSPOT:PAIR-LTC-USDT,BNBSPOT:PAIR-XRP-USDT,BNBSPOT:PAIR-USDC-USDT" Cmd="${Cmd} --credentials_file=/.creds" @@ -47,15 +54,26 @@ elif [ "${job}" == "BNBSPOT_CLD21" ] ; then Cmd="${Cmd} --admin_port=7203" Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBSPOT.CLD21.log" elif [ "${job}" == "BNBSPOT_CVTTDATA" ] ; then - Cmd="python3.10" - Cmd="${Cmd} cvttpy/apps/md/md_recorder.py" - Cmd="${Cmd} --config=http://${ConfigServer}:6789/apps/md_recorder" Cmd="${Cmd} --active_exchanges=BNBSPOT" Cmd="${Cmd} --instruments=BNBSPOT:PAIR-BTC-USDT,BNBSPOT:PAIR-ETH-USDT,BNBSPOT:PAIR-LTC-USDT,BNBSPOT:PAIR-XRP-USDT,BNBSPOT:PAIR-USDC-USDT" Cmd="${Cmd} --credentials_file=/.creds" Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA" Cmd="${Cmd} --admin_port=7204" Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBSPOT.CVTTDATA.log" +elif [ "${job}" == "BNBFUT_CVTTDATA" ] ; then + Cmd="${Cmd} --active_exchanges=BNBFUT" + Cmd="${Cmd} --instruments=BNBFUT:PERP-BTC-USDT,BNBFUT:PERP-ETH-USDT" + Cmd="${Cmd} --credentials_file=/.creds" + Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA" + Cmd="${Cmd} --admin_port=7205" + Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBFUT.CVTTDATA.log" +elif [ "${job}" == "COINBASE_CVTTDATA" ] ; then + Cmd="${Cmd} --active_exchanges=COINBASE_AT" + Cmd="${Cmd} --instruments=COINBASE:PAIR-BTC-USD,COINBASE:PAIR-ETH-USD,COINBASE:PAIR-LTC-USD,COINBASE:PAIR-XRP-USD" + Cmd="${Cmd} --credentials_file=/.creds" + Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA" + Cmd="${Cmd} --admin_port=7206" + Cmd="${Cmd} --log_file=/logs/%T.MD_REC.COINBASE.CVTTDATA.log" else echo "Unrecognized JOB: ${job}" exit 1 diff --git a/market_data/md_recorder_monitor/.creds b/market_data/md_recorder_monitor/.creds new file mode 100644 index 0000000..345503a --- /dev/null +++ b/market_data/md_recorder_monitor/.creds @@ -0,0 +1,22 @@ +{ + "__dummy__": null + , "MATTERMOST": { + "url": "https://mattermost.cryptovaltrading.com" + , "team": "CVTT" + , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano" + } + , "TSDB_MD_CLD21": { + "host": "cloud21.cvtt.vpn" + , "port": 5432 + , "user": "cvtt" + , "database": "cvtt_md" + , "password": "ICdIh0JnMM7vM7Pf" + } + , "TSDB_MD_CVTTDATA": { + "host": "cvttdata.cvtt.vpn" + , "port": 5432 + , "user": "cvtt" + , "database": "cvtt_md" + , "password": "ICdIh0JnMM7vM7Pf" + } +} diff --git a/market_data/md_recorder_monitor/Dockerfile b/market_data/md_recorder_monitor/Dockerfile new file mode 100644 index 0000000..e8f14c5 --- /dev/null +++ b/market_data/md_recorder_monitor/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.10-slim + +ARG FROM_DIR=docker_dev/market_data/md_recorder_monitor + + +COPY ${FROM_DIR}/requirements.txt / +RUN pip install --upgrade pip --root-user-action=ignore +RUN pip install -r /requirements.txt --root-user-action=ignore + +COPY cvttpy /cvttpy +COPY ${FROM_DIR}/.creds /.creds +COPY ${FROM_DIR}/run_mdr_monitor.sh /run_mdr_monitor.sh + + +# Shared Volumes +RUN mkdir -p /app/data +RUN mkdir /logs +RUN chmod +x /run_mdr_monitor.sh + +WORKDIR / +SHELL ["/bin/bash", "-c"] +ENV PYTHONPATH=/ +ENTRYPOINT [ "/run_mdr_monitor.sh" ] + diff --git a/market_data/md_recorder_monitor/requirements.txt b/market_data/md_recorder_monitor/requirements.txt new file mode 100644 index 0000000..3045f17 --- /dev/null +++ b/market_data/md_recorder_monitor/requirements.txt @@ -0,0 +1,7 @@ +aiohttp>=3.7.4.post0 +nest-asyncio>=1.5.5 +asyncpg>=0.27.0 +hjson>=3.1.0 +pandas>=1.5.3 +sortedcontainers>=2.4.0 +aioredis>=2.0.1 \ No newline at end of file diff --git a/market_data/md_recorder_monitor/run_mdr_monitor.sh b/market_data/md_recorder_monitor/run_mdr_monitor.sh new file mode 100755 index 0000000..d4c9c13 --- /dev/null +++ b/market_data/md_recorder_monitor/run_mdr_monitor.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# runs in container + +usage() { + echo "Usage: $0 []" + exit 1 +} + +ValidInstances=('CLD21' 'CVTTDATA') +is_valid() { + local inst=$1 + for valid_inst in "${ValidInstances[@]}"; + do + if [[ "$inst" == "$valid_inst" ]]; then + return 0 + fi + done + return 1 +} + + +Instance=${1} +if ! is_valid "${Instance}"; then + usage +fi + +AdminPort=7225 +if [ "${2}" != "" ]; then + AdminPort=${2} +fi + + + +# ConfigServer=cloud16.cvtt.vpn +ConfigServer=cloud23.cvtt.vpn:6789 + +Cmd="python3.10" +Cmd="${Cmd} cvttpy/apps/md/md_rec_monitor.py" +Cmd="${Cmd} --config=http://c${ConfigServer}/apps/md_rec_monitor" +Cmd="${Cmd} --db=${Instance}" +Cmd="${Cmd} --admin_port=${AdminPort}" +Cmd="${Cmd} --log_file=/logs/%%T.MD_REC.MONITOR.${Instance}.log + + +echo ${Cmd} +eval ${Cmd} + +