diff --git a/VERSION b/VERSION index 1c3cdd1..80db2c2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.4.FX1,exch_acct_cfgname fixes \ No newline at end of file +0.2.6,improved build.sh for microservices \ No newline at end of file diff --git a/microservices/.env b/microservices/.env new file mode 100644 index 0000000..412fd15 --- /dev/null +++ b/microservices/.env @@ -0,0 +1,18 @@ +CVTT_USER=1001:1001 +CVTT_VERSION=0.0.3 + +# MDGW_CONFIG_FILE=config_musvc.cfg +# MDGW_CREDS_FILE=.creds +MDGW_LOG_FILE=%D.md_gateway.log +MDGW_ADD_ARGS= + +# OEGW_CONFIG_FILE=config_musvc.cfg +# OEGW_CREDS_FILE=.creds +OEGW_LOG_FILE=%D.oe_gateway.log +#OEGW_ADD_ARGS="--log_level=DEBUG --debug_mode" +OEGW_ADD_ARGS="--debug_mode" + +# ACGW_CONFIG_FILE=config_musvc.cfg +# ACGW_CREDS_FILE=.creds +ACGW_LOG_FILE=%D.ac_gateway.log +ACGW_ADD_ARGS= diff --git a/microservices/tester/.dockerignore b/microservices/ac_gateway/.dockerignore similarity index 100% rename from microservices/tester/.dockerignore rename to microservices/ac_gateway/.dockerignore diff --git a/microservices/ac_gateway/Dockerfile b/microservices/ac_gateway/Dockerfile new file mode 100644 index 0000000..33e3bc2 --- /dev/null +++ b/microservices/ac_gateway/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.12-slim + +COPY requirements.txt / + +RUN pip install --upgrade pip --root-user-action=ignore +RUN pip install -r /requirements.txt --root-user-action=ignore + +COPY cvttpy /cvttpy + +RUN mkdir -p /logs /config + +SHELL ["/bin/bash", "-c"] + +ENV PYTHONPATH=/ + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/microservices/ac_gateway/entrypoint.sh b/microservices/ac_gateway/entrypoint.sh new file mode 100755 index 0000000..5c6e64c --- /dev/null +++ b/microservices/ac_gateway/entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +ConfigDir=/config +LogDir=/logs + +Cmd="python3.12" +Cmd+=" cvttpy/apps/microservices/market_gateways/accounting_gateway.py" +Cmd+=" --config=${ConfigDir}/${ACGW_CONFIG_FILE:-cvtt_musvc.cfg}" +Cmd+=" --credentials_file=${ConfigDir}/${ACGW_CREDS_FILE:-.creds}" +Cmd+=" --log_file=${LogDir}/${ACGW_LOG_FILE:-%T.ac_gateway.log}" +Cmd+=" ${ACGW_ADD_ARGS}" +echo ${Cmd} +exec ${Cmd} + diff --git a/microservices/requirements.txt b/microservices/ac_gateway/requirements.txt similarity index 80% rename from microservices/requirements.txt rename to microservices/ac_gateway/requirements.txt index 852e647..e840a84 100644 --- a/microservices/requirements.txt +++ b/microservices/ac_gateway/requirements.txt @@ -5,3 +5,5 @@ sortedcontainers>=2.4.0 redis>=5.0.8 python-dateutil>=2.8.2 types-python-dateutil>=2.8.19.6 +cryptography>=43.0.0 +PyJWT>=2.10.1 diff --git a/microservices/build.sh b/microservices/build.sh new file mode 100755 index 0000000..afa16ab --- /dev/null +++ b/microservices/build.sh @@ -0,0 +1,79 @@ +#!/bin/bash + + +usage() { + echo "Usage: $0 -I -V -L :" + exit 1 +} + +# --- Settings +RegistryService=cloud21.cvtt.vpn:5500 +# --- Settings + + +# ---------------- cmdline +while getopts "I:V:L:" opt; do + case ${opt} in + I ) + ImageName=$OPTARG + ;; + V ) + ImageVersion=$OPTARG + ;; + L ) + Library=$OPTARG + ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + usage + ;; + : ) + echo "Option -$OPTARG requires an argument." >&2 + usage + ;; + esac +done +# ---------------- cmdline + +if [ "${ImageName}" == "" ] +then + usage +fi + +if [ "${ImageVersion}" == "" ] +then + usage +fi + +Project= +ProjectVersion= +if [ "${Library}" != "" ] ; then + IFS=':' read -ra parts <<< "${Library}" + Project=${parts[0]} + ProjectVersion=${parts[1]} +fi +# exit + +DockerDir=$(realpath $(dirname ${0})/${ImageName}) + +cd ${DockerDir} + +source ../mu_svc_functions.sh +if [ "${Project}" != "" ]; then + get_project_version ${Project} ${ProjectVersion} +fi + + +function cleanup { + cd ${DockerDir} + rm -rf cvttpy +} +trap cleanup EXIT + +build_docker_image ${ImageName} ${RegistryService} ${ImageVersion} ${Project} ${ProjectVersion} + + +# echo "**** D E B U G E X I T" && exit +echo "***** ${0} D O N E" + + diff --git a/microservices/docker-compose.yml b/microservices/docker-compose.yml index 7fe47c6..9ce5e26 100644 --- a/microservices/docker-compose.yml +++ b/microservices/docker-compose.yml @@ -1,36 +1,60 @@ -version: "3.9" - +# CVTT Microservices services: md_gateway: - image: cloud21.cvtt.vpn:5500/md_gateway:latest + image: cloud21.cvtt.vpn:5500/md_gateway:${CVTT_VERSION} container_name: md_gateway + pull_policy: always + user: ${CVTT_USER:-1001:1001} environment: - - REDIS_HOST=redis + CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789} + MDGW_CONFIG_FILE: ${MDGW_CONFIG_FILE:-cvtt_musvc.cfg} + MDGW_CREDS_FILE: ${MDGW_CONFIG_FILE:-.creds} + MDGW_LOG_FILE: ${MDGW_LOG_FILE:-%T.md_gateway.log} + MDGW_ADD_ARGS: ${MDGW_ADD_ARGS:-} volumes: - - ./.creds:/.creds - ./config:/config - ./logs:/logs - - ./data:/data depends_on: - redis - tester: - image: cloud21.cvtt.vpn:5500/tester:latest - container_name: tester + oe_gateway: + image: cloud21.cvtt.vpn:5500/oe_gateway:${CVTT_VERSION} + container_name: oe_gateway + pull_policy: always + user: ${CVTT_USER:-1001:1001} environment: - - REDIS_HOST=redis + CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789} + OEGW_CONFIG_FILE: ${OEGW_CONFIG_FILE:-cvtt_musvc.cfg} + OEGW_CREDS_FILE: ${OEGW_CONFIG_FILE:-.creds} + OEGW_LOG_FILE: ${OEGW_LOG_FILE:-%T.oe_gateway.log} + OEGW_ADD_ARGS: ${OEGW_ADD_ARGS:-} + volumes: + - ./config:/config + - ./logs:/logs + depends_on: + - redis + + ac_gateway: + image: cloud21.cvtt.vpn:5500/ac_gateway:${CVTT_VERSION} + container_name: ac_gateway + pull_policy: always + user: ${CVTT_USER:-1001:1001} + environment: + CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789} + ACGW_CONFIG_FILE: ${ACGW_CONFIG_FILE:-cvtt_musvc.cfg} + ACGW_CREDS_FILE: ${ACGW_CONFIG_FILE:-.creds} + ACGW_LOG_FILE: ${ACGW_LOG_FILE:-%T.ac_gateway.log} + ACGW_ADD_ARGS: ${ACGW_ADD_ARGS:-} volumes: - ./config:/config - ./logs:/logs - - ./data:/data depends_on: - redis - - md_gateway redis: image: redis:latest - container_name: redis + container_name: redis-cvtt ports: - - "6379:6379" + - "16379:6379" volumes: - ./data/redis:/data diff --git a/microservices/md_gateway/Dockerfile b/microservices/md_gateway/Dockerfile index 5a421a3..33e3bc2 100644 --- a/microservices/md_gateway/Dockerfile +++ b/microservices/md_gateway/Dockerfile @@ -1,25 +1,19 @@ FROM python:3.12-slim -ARG ROOT_MCRSVC_DIR=docker_dev/microservices -ARG FROM_DIR=${ROOT_MCRSVC_DIR}/md_gateway +COPY requirements.txt / -COPY ${ROOT_MCRSVC_DIR}/requirements.txt / RUN pip install --upgrade pip --root-user-action=ignore RUN pip install -r /requirements.txt --root-user-action=ignore COPY cvttpy /cvttpy -# COPY ${ROOT_MCRSVC_DIR}/.creds /.creds +RUN mkdir -p /logs /config -COPY ${FROM_DIR}/run_mdgw.sh /run_mdgw.sh -RUN chmod +x /run_mdgw.sh - -RUN mkdir /logs -RUN mkdir -p /config - - -WORKDIR / SHELL ["/bin/bash", "-c"] -ENV PYTHONPATH=/ -ENTRYPOINT [ "/run_mdgw.sh" ] +ENV PYTHONPATH=/ + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/microservices/md_gateway/entrypoint.sh b/microservices/md_gateway/entrypoint.sh new file mode 100755 index 0000000..69ed4ec --- /dev/null +++ b/microservices/md_gateway/entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +ConfigDir=/config +LogDir=/logs + +Cmd="python3.12" +Cmd+=" cvttpy/apps/microservices/market_gateways/md_gateway.py" +Cmd+=" --config=${ConfigDir}/${MDGW_CONFIG_FILE:-cvtt_musvc.cfg}" +Cmd+=" --credentials_file=${ConfigDir}/${MDGW_CREDS_FILE:-.creds}" +Cmd+=" --log_file=${LogDir}/${MDGW_LOG_FILE:-%T.md_gateway.log}" +Cmd+=" ${MDGW_ADD_ARGS}" +echo ${Cmd} +exec ${Cmd} + diff --git a/microservices/md_gateway/requirements.txt b/microservices/md_gateway/requirements.txt new file mode 100644 index 0000000..e840a84 --- /dev/null +++ b/microservices/md_gateway/requirements.txt @@ -0,0 +1,9 @@ +aiohttp>=3.7.4.post0 +nest-asyncio>=1.5.5 +hjson>=3.1.0 +sortedcontainers>=2.4.0 +redis>=5.0.8 +python-dateutil>=2.8.2 +types-python-dateutil>=2.8.19.6 +cryptography>=43.0.0 +PyJWT>=2.10.1 diff --git a/microservices/md_gateway/run_mdgw.sh b/microservices/md_gateway/run_mdgw.sh deleted file mode 100755 index 95c11ca..0000000 --- a/microservices/md_gateway/run_mdgw.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -# runs in container - -# runs on host to start container -usage() { - echo -n "Usage: ${0}" - echo -n " [-c ]" - echo -n " [-a ]" - echo -n " [-z (compress log)"] - echo - exit 1 -} - -Config=/config/cvtt.cfg -AdminPort=7220 -COMPRESS_LOG=false - - -while getopts "c:a:z" opt; do - case ${opt} in - c ) Config=$OPTARG ;; - a ) AdminPort=$OPTARG ;; - z ) COMPRESS_LOG=true ;; - \? ) - echo "Invalid option: -$OPTARG" >&2 - usage - ;; - : ) - echo "Option -$OPTARG requires an argument." >&2 - usage - ;; - esac -done - -Cmd="python3.12" -Cmd="${Cmd} cvttpy/apps/microservices/market_gateways/md_gateway.py" -Cmd="${Cmd} --config=${Config}" -Cmd="${Cmd} --credentials_file=/.creds" -Cmd="${Cmd} --admin_port=${AdminPort}" -Cmd="${Cmd} --log_file=/logs/%T.md_gateway.log" -if ${COMPRESS_LOG} ; then - Cmd="${Cmd} --compress_log" -fi -echo ${Cmd} -eval ${Cmd} - - diff --git a/microservices/mu_svc_functions.sh b/microservices/mu_svc_functions.sh new file mode 100755 index 0000000..c9e3351 --- /dev/null +++ b/microservices/mu_svc_functions.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +get_project_version() { + Project=${1} + Version=${2} + + host="cloud21.cvtt.vpn" + port="22" + user="cvttdist" + rel_dir="/home/cvttdist/software/cvtt2" + + if [ "${Version}" == "latest" ]; then + echo "Checking for latest version of ${Project} on ${user}@${host}:${rel_dir}" + Version=$(ssh -q -p ${port} ${user}@${host} "ls -tr ${rel_dir}/${Project} | tail -1" ) + echo "Latest version is ${Version}" + fi + echo "Checking ${user}@${host} for ${rel_dir}/${Project}/${Version} ..." + if ssh -q -p ${port} ${user}@${host} "test -d ${rel_dir}/${Project}/${Version}" + then + echo "Version ${Version} found..." + rsync_cmd="rsync -ahv -e \"ssh -p ${port}\"" + rsync_cmd="${rsync_cmd} ${user}@${host}:${rel_dir}/${Project}/${Version}/" + rsync_cmd="${rsync_cmd} ./" + echo ${rsync_cmd} + eval ${rsync_cmd} + status=$? + if [ ${status} -eq 0 ] + then + echo "Loading successful..." + return 0 + else + return 1 + fi + else + echo "Not Found ${rel_dir}/${Project}/${Version} on ${user}@${host}" + return 1 + fi +} + +build_docker_image() { + ImageName=${1} + RegistryService=${2} + Version=${3} + Project=${4} + ProjectVersion=${5} + + Cmd="docker build" + if [ "${ProjectVersion}" != "" ]; then + Cmd+=" --label ${Project}=\"${ProjectVersion}\"" + fi + Cmd+=" -t ${ImageName}" + Cmd+=" -t ${ImageName}:latest" + Cmd+=" -t ${ImageName}:${Version}" + Cmd+=" -f Dockerfile" + Cmd+=" ." + echo ${Cmd} + eval ${Cmd} || exit + + Cmd="docker tag ${ImageName}:latest ${RegistryService}/${ImageName}:latest" + echo ${Cmd} + eval ${Cmd} || exit + + Cmd="docker tag ${ImageName}:${Version} ${RegistryService}/${ImageName}:${Version}" + echo ${Cmd} + eval ${Cmd} || exit + + Cmd="docker push ${RegistryService}/${ImageName}:latest" + echo ${Cmd} + eval ${Cmd} || exit + + Cmd="docker push ${RegistryService}/${ImageName}:${Version}" + echo ${Cmd} + eval ${Cmd} || exit +} + + diff --git a/microservices/oe_gateway/Dockerfile b/microservices/oe_gateway/Dockerfile index 3a6649e..33e3bc2 100644 --- a/microservices/oe_gateway/Dockerfile +++ b/microservices/oe_gateway/Dockerfile @@ -1,25 +1,19 @@ FROM python:3.12-slim -ARG ROOT_MCRSVC_DIR=docker_dev/microservices -ARG FROM_DIR=${ROOT_MCRSVC_DIR}/oe_gateway +COPY requirements.txt / -COPY ${ROOT_MCRSVC_DIR}/requirements.txt / RUN pip install --upgrade pip --root-user-action=ignore RUN pip install -r /requirements.txt --root-user-action=ignore COPY cvttpy /cvttpy -COPY ${ROOT_MCRSVC_DIR}/.creds /.creds +RUN mkdir -p /logs /config -COPY ${FROM_DIR}/run_oegw.sh /run_oegw.sh -RUN chmod +x /run_oegw.sh - -RUN mkdir /logs -RUN mkdir -p /config - - -WORKDIR / SHELL ["/bin/bash", "-c"] -ENV PYTHONPATH=/ -ENTRYPOINT [ "/run_oegw.sh" ] +ENV PYTHONPATH=/ + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/microservices/oe_gateway/entrypoint.sh b/microservices/oe_gateway/entrypoint.sh new file mode 100755 index 0000000..af08fa4 --- /dev/null +++ b/microservices/oe_gateway/entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +ConfigDir=/config +LogDir=/logs + +Cmd="python3.12" +Cmd+=" cvttpy/apps/microservices/market_gateways/order_entry_gateway.py" +Cmd+=" --config=${ConfigDir}/${OEGW_CONFIG_FILE:-cvtt_musvc.cfg}" +Cmd+=" --credentials_file=${ConfigDir}/${OEGW_CREDS_FILE:-.creds}" +Cmd+=" --log_file=${LogDir}/${OEGW_LOG_FILE:-%T.oe_gateway.log}" +Cmd+=" ${OEGW_ADD_ARGS}" +echo ${Cmd} +exec ${Cmd} + diff --git a/microservices/oe_gateway/requirements.txt b/microservices/oe_gateway/requirements.txt new file mode 100644 index 0000000..d23f46d --- /dev/null +++ b/microservices/oe_gateway/requirements.txt @@ -0,0 +1,11 @@ +aiohttp>=3.7.4.post0 +nest-asyncio>=1.5.5 +hjson>=3.1.0 +sortedcontainers>=2.4.0 +redis>=5.0.8 +python-dateutil>=2.8.2 +types-python-dateutil>=2.8.19.6 +cryptography>=43.0.0 +PyJWT>=2.10.1 + + diff --git a/microservices/oe_gateway/run_oegw.sh b/microservices/oe_gateway/run_oegw.sh deleted file mode 100755 index af7f658..0000000 --- a/microservices/oe_gateway/run_oegw.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -# runs in container - -# runs on host to start container -usage() { - echo -n "Usage: ${0}" - echo -n " [-c ]" - echo -n " [-a ]" - echo -n " [-z (compress log)"] - echo - exit 1 -} - -Config=/config/cvtt.cfg -AdminPort=7220 -COMPRESS_LOG=false - - -while getopts "c:a:z" opt; do - case ${opt} in - c ) Config=$OPTARG ;; - a ) AdminPort=$OPTARG ;; - z ) COMPRESS_LOG=true ;; - \? ) - echo "Invalid option: -$OPTARG" >&2 - usage - ;; - : ) - echo "Option -$OPTARG requires an argument." >&2 - usage - ;; - esac -done - -Cmd="python3.12" -Cmd="${Cmd} cvttpy/apps/microservices/market_gateways/order_entry_gateway.py" -Cmd="${Cmd} --config=${Config}" -Cmd="${Cmd} --credentials_file=/.creds" -Cmd="${Cmd} --admin_port=${AdminPort}" -Cmd="${Cmd} --log_file=/logs/%T.md_gateway.log" -if ${COMPRESS_LOG} ; then - Cmd="${Cmd} --compress_log" -fi -echo ${Cmd} -eval ${Cmd} - - diff --git a/microservices/tester/Dockerfile b/microservices/tester/Dockerfile deleted file mode 100644 index 4f8e8ed..0000000 --- a/microservices/tester/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -FROM python:3.12-slim - -ARG ROOT_MUSVC_DIR=docker_dev/microservices -ARG FROM_DIR=${ROOT_MUSVC_DIR}/tester - -COPY ${ROOT_MUSVC_DIR}/requirements.txt / -RUN pip install --upgrade pip --root-user-action=ignore -RUN pip install -r /requirements.txt --root-user-action=ignore - -COPY cvttpy /cvttpy -# COPY ${ROOT_MUSVC_DIR}/.creds /.creds - - -COPY ${FROM_DIR}/run_tester.sh /run_tester.sh -RUN chmod +x /run_tester.sh - -RUN mkdir /logs -RUN mkdir -p /config -RUN mkdir -p /data - - -WORKDIR / -SHELL ["/bin/bash", "-c"] -ENV PYTHONPATH=/ -ENTRYPOINT [ "/run_tester.sh" ] - diff --git a/microservices/tester/run_tester.sh b/microservices/tester/run_tester.sh deleted file mode 100755 index 748ff4a..0000000 --- a/microservices/tester/run_tester.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -# runs in container - -# runs on host to start container -usage() { - echo -n "Usage: ${0}" - echo -n " [-c ]" - echo -n " [-i ]" - echo -n " [-d ]" - echo -n " [-a ]" - echo -n " [-z (compress log (false))"] - echo - exit 1 -} - -Config=/config/cvtt.cfg -AdminPort=7221 -COMPRESS_LOG=false -Instruments="COINBASE:PAIR-BTC-USD,BNBFUT:PERP-BTC-USDT" -DataTypes="TRADES" - -while getopts "a:c:d:i:z" opt; do - case ${opt} in - c ) Config=$OPTARG ;; - a ) AdminPort=$OPTARG ;; - i ) Instruments=$OPTARG ;; - d ) DataTypes=$OPTARG ;; - z ) COMPRESS_LOG=true ;; - \? ) - echo "Invalid option: -$OPTARG" >&2 - usage - ;; - : ) - echo "Option -$OPTARG requires an argument." >&2 - usage - ;; - esac -done - -Cmd="python3.12" -Cmd="${Cmd} cvttpy/apps/microservices/tester/musvc_tester.py" -Cmd="${Cmd} --config=${Config}" -Cmd="${Cmd} --credentials_file=/.creds" -Cmd="${Cmd} --admin_port=${AdminPort}" -Cmd="${Cmd} --data_types=${DataTypes}" -Cmd="${Cmd} --instruments=${Instruments}" -Cmd="${Cmd} --log_file=/logs/%T.tester.log" -if ${COMPRESS_LOG} ; then - Cmd="${Cmd} --compress_log" -fi -echo ${Cmd} -eval ${Cmd} - -