merge

VERSION

@@ -1 +1 @@
-0.2.4.FX1,exch_acct_cfgname fixes
+0.2.6,improved build.sh for microservices

microservices/.env

@@ -0,0 +1,18 @@
+CVTT_USER=1001:1001
+CVTT_VERSION=0.0.3
+
+# MDGW_CONFIG_FILE=config_musvc.cfg
+# MDGW_CREDS_FILE=.creds
+MDGW_LOG_FILE=%D.md_gateway.log
+MDGW_ADD_ARGS=
+
+# OEGW_CONFIG_FILE=config_musvc.cfg
+# OEGW_CREDS_FILE=.creds
+OEGW_LOG_FILE=%D.oe_gateway.log
+#OEGW_ADD_ARGS="--log_level=DEBUG --debug_mode"
+OEGW_ADD_ARGS="--debug_mode"
+
+# ACGW_CONFIG_FILE=config_musvc.cfg
+# ACGW_CREDS_FILE=.creds
+ACGW_LOG_FILE=%D.ac_gateway.log
+ACGW_ADD_ARGS=

microservices/ac_gateway/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.12-slim
+
+COPY requirements.txt /
+
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+
+RUN mkdir -p /logs /config
+
+SHELL ["/bin/bash", "-c"]
+
+ENV PYTHONPATH=/
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

microservices/ac_gateway/entrypoint.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+ConfigDir=/config
+LogDir=/logs
+
+Cmd="python3.12"
+Cmd+=" cvttpy/apps/microservices/market_gateways/accounting_gateway.py"
+Cmd+=" --config=${ConfigDir}/${ACGW_CONFIG_FILE:-cvtt_musvc.cfg}"
+Cmd+=" --credentials_file=${ConfigDir}/${ACGW_CREDS_FILE:-.creds}"
+Cmd+=" --log_file=${LogDir}/${ACGW_LOG_FILE:-%T.ac_gateway.log}"
+Cmd+=" ${ACGW_ADD_ARGS}"
+echo ${Cmd}
+exec ${Cmd}
+

microservices/ac_gateway/requirements.txt

@@ -5,3 +5,5 @@ sortedcontainers>=2.4.0
 redis>=5.0.8
 python-dateutil>=2.8.2
 types-python-dateutil>=2.8.19.6
+cryptography>=43.0.0
+PyJWT>=2.10.1

microservices/build.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+
+usage() {
+  echo "Usage: $0 -I <image_name> -V <image_version> -L <library>:<library_version>"
+  exit 1
+}
+
+# --- Settings
+RegistryService=cloud21.cvtt.vpn:5500
+# --- Settings
+
+
+# ---------------- cmdline
+while getopts "I:V:L:" opt; do
+  case ${opt} in
+    I )
+      ImageName=$OPTARG
+      ;;
+    V )
+      ImageVersion=$OPTARG
+      ;;
+    L )
+      Library=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+# ---------------- cmdline
+
+if [ "${ImageName}" == "" ]
+then
+    usage
+fi
+
+if [ "${ImageVersion}" == "" ]
+then
+    usage
+fi
+
+Project=
+ProjectVersion=
+if  [ "${Library}" != "" ] ; then
+    IFS=':' read -ra parts <<< "${Library}"
+    Project=${parts[0]}
+    ProjectVersion=${parts[1]}
+fi
+# exit
+
+DockerDir=$(realpath $(dirname ${0})/${ImageName})
+
+cd ${DockerDir}
+
+source ../mu_svc_functions.sh
+if [ "${Project}" != "" ]; then
+    get_project_version ${Project} ${ProjectVersion}
+fi
+
+
+function cleanup {
+    cd ${DockerDir}
+    rm -rf cvttpy
+}
+trap cleanup EXIT
+
+build_docker_image ${ImageName} ${RegistryService} ${ImageVersion} ${Project} ${ProjectVersion}
+
+
+# echo "**** D E B U G  E X I T" && exit
+echo "***** ${0}  D O N E"
+
+

microservices/docker-compose.yml

@@ -1,36 +1,60 @@
-version: "3.9"
+# CVTT Microservices
-
 services:
     md_gateway:
-        image: cloud21.cvtt.vpn:5500/md_gateway:latest
+        image: cloud21.cvtt.vpn:5500/md_gateway:${CVTT_VERSION}
         container_name: md_gateway
+        pull_policy: always
+        user: ${CVTT_USER:-1001:1001}
         environment:
-          - REDIS_HOST=redis
+          CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          MDGW_CONFIG_FILE:  ${MDGW_CONFIG_FILE:-cvtt_musvc.cfg}
+          MDGW_CREDS_FILE:  ${MDGW_CONFIG_FILE:-.creds}
+          MDGW_LOG_FILE:  ${MDGW_LOG_FILE:-%T.md_gateway.log}
+          MDGW_ADD_ARGS: ${MDGW_ADD_ARGS:-}
         volumes:
-          - ./.creds:/.creds
           - ./config:/config
           - ./logs:/logs
-          - ./data:/data
         depends_on:
           - redis
 
-    tester:
+    oe_gateway:
-        image: cloud21.cvtt.vpn:5500/tester:latest
+        image: cloud21.cvtt.vpn:5500/oe_gateway:${CVTT_VERSION}
-        container_name: tester
+        container_name: oe_gateway
+        pull_policy: always
+        user: ${CVTT_USER:-1001:1001}
         environment:
-          - REDIS_HOST=redis
+          CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          OEGW_CONFIG_FILE:  ${OEGW_CONFIG_FILE:-cvtt_musvc.cfg}
+          OEGW_CREDS_FILE:  ${OEGW_CONFIG_FILE:-.creds}
+          OEGW_LOG_FILE:  ${OEGW_LOG_FILE:-%T.oe_gateway.log}
+          OEGW_ADD_ARGS: ${OEGW_ADD_ARGS:-}
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+        depends_on:
+          - redis
+
+    ac_gateway:
+        image: cloud21.cvtt.vpn:5500/ac_gateway:${CVTT_VERSION}
+        container_name: ac_gateway
+        pull_policy: always
+        user: ${CVTT_USER:-1001:1001}
+        environment:
+          CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          ACGW_CONFIG_FILE:  ${ACGW_CONFIG_FILE:-cvtt_musvc.cfg}
+          ACGW_CREDS_FILE:  ${ACGW_CONFIG_FILE:-.creds}
+          ACGW_LOG_FILE:  ${ACGW_LOG_FILE:-%T.ac_gateway.log}
+          ACGW_ADD_ARGS: ${ACGW_ADD_ARGS:-}
         volumes:
           - ./config:/config
           - ./logs:/logs
-          - ./data:/data
         depends_on:
           - redis
-          - md_gateway
 
     redis:
         image: redis:latest
-        container_name: redis
+        container_name: redis-cvtt
         ports:
-          - "6379:6379"
+          - "16379:6379"
         volumes:
           - ./data/redis:/data

microservices/md_gateway/Dockerfile

@@ -1,25 +1,19 @@
 FROM python:3.12-slim
 
-ARG ROOT_MCRSVC_DIR=docker_dev/microservices
+COPY requirements.txt /
-ARG FROM_DIR=${ROOT_MCRSVC_DIR}/md_gateway
 
-COPY ${ROOT_MCRSVC_DIR}/requirements.txt /
 RUN pip install --upgrade pip --root-user-action=ignore
 RUN pip install -r /requirements.txt --root-user-action=ignore
 
 COPY cvttpy /cvttpy
-# COPY ${ROOT_MCRSVC_DIR}/.creds  /.creds
 
+RUN mkdir -p /logs /config
 
-COPY ${FROM_DIR}/run_mdgw.sh /run_mdgw.sh
-RUN chmod +x /run_mdgw.sh
-
-RUN mkdir /logs
-RUN mkdir -p /config
-
-
-WORKDIR /
 SHELL ["/bin/bash", "-c"]
-ENV PYTHONPATH=/
-ENTRYPOINT [ "/run_mdgw.sh" ]
 
+ENV PYTHONPATH=/
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

microservices/md_gateway/entrypoint.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+ConfigDir=/config
+LogDir=/logs
+
+Cmd="python3.12"
+Cmd+=" cvttpy/apps/microservices/market_gateways/md_gateway.py"
+Cmd+=" --config=${ConfigDir}/${MDGW_CONFIG_FILE:-cvtt_musvc.cfg}"
+Cmd+=" --credentials_file=${ConfigDir}/${MDGW_CREDS_FILE:-.creds}"
+Cmd+=" --log_file=${LogDir}/${MDGW_LOG_FILE:-%T.md_gateway.log}"
+Cmd+=" ${MDGW_ADD_ARGS}"
+echo ${Cmd}
+exec ${Cmd}
+

microservices/md_gateway/requirements.txt

@@ -0,0 +1,9 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5
+hjson>=3.1.0
+sortedcontainers>=2.4.0
+redis>=5.0.8
+python-dateutil>=2.8.2
+types-python-dateutil>=2.8.19.6
+cryptography>=43.0.0
+PyJWT>=2.10.1

microservices/md_gateway/run_mdgw.sh

@@ -1,48 +0,0 @@
-#!/bin/bash
-
-# runs in container
-
-# runs on host to start container
-usage() {
-    echo -n "Usage: ${0}"
-    echo -n " [-c <config (/config/cvtt.cfg)>]"
-    echo -n " [-a <admin_port (7220)>]"
-    echo -n " [-z (compress log)"]
-    echo
-    exit 1
-}
-
-Config=/config/cvtt.cfg
-AdminPort=7220
-COMPRESS_LOG=false
-
-
-while getopts "c:a:z" opt; do
-    case ${opt} in
-        c ) Config=$OPTARG  ;;
-        a ) AdminPort=$OPTARG ;;
-        z ) COMPRESS_LOG=true ;;
-        \? )
-            echo "Invalid option: -$OPTARG" >&2
-            usage
-            ;;
-        : )
-            echo "Option -$OPTARG requires an argument." >&2
-            usage
-            ;;
-    esac
-done
-
-Cmd="python3.12"
-Cmd="${Cmd} cvttpy/apps/microservices/market_gateways/md_gateway.py"
-Cmd="${Cmd} --config=${Config}"
-Cmd="${Cmd} --credentials_file=/.creds"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_file=/logs/%T.md_gateway.log"
-if ${COMPRESS_LOG} ; then
-    Cmd="${Cmd} --compress_log"
-fi
-echo ${Cmd}
-eval ${Cmd}
-
-

microservices/mu_svc_functions.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+get_project_version() {
+    Project=${1}
+    Version=${2}
+
+    host="cloud21.cvtt.vpn"
+    port="22"
+    user="cvttdist"
+    rel_dir="/home/cvttdist/software/cvtt2"
+
+    if [ "${Version}" == "latest" ]; then
+        echo "Checking for latest version of ${Project} on ${user}@${host}:${rel_dir}"
+        Version=$(ssh -q -p ${port} ${user}@${host} "ls -tr ${rel_dir}/${Project} | tail -1" )
+        echo "Latest version is ${Version}"
+    fi
+    echo "Checking ${user}@${host} for ${rel_dir}/${Project}/${Version} ..."
+    if ssh -q -p ${port} ${user}@${host} "test -d ${rel_dir}/${Project}/${Version}"
+    then
+        echo "Version ${Version} found..."
+        rsync_cmd="rsync -ahv -e \"ssh -p ${port}\""
+        rsync_cmd="${rsync_cmd} ${user}@${host}:${rel_dir}/${Project}/${Version}/"
+        rsync_cmd="${rsync_cmd} ./"
+        echo ${rsync_cmd}
+        eval ${rsync_cmd}
+        status=$?
+        if [ ${status} -eq 0 ]
+        then
+            echo "Loading successful..."
+            return 0
+        else
+            return 1
+        fi
+    else
+        echo "Not Found ${rel_dir}/${Project}/${Version} on ${user}@${host}"
+        return 1
+    fi
+}
+
+build_docker_image() {
+    ImageName=${1}
+    RegistryService=${2}
+    Version=${3}
+    Project=${4}
+    ProjectVersion=${5}
+
+    Cmd="docker build"
+    if [ "${ProjectVersion}" != "" ]; then
+        Cmd+=" --label ${Project}=\"${ProjectVersion}\""
+    fi
+    Cmd+=" -t ${ImageName}"
+    Cmd+=" -t ${ImageName}:latest"
+    Cmd+=" -t ${ImageName}:${Version}"
+    Cmd+=" -f Dockerfile"
+    Cmd+=" ."
+    echo ${Cmd}
+    eval ${Cmd} || exit
+
+    Cmd="docker tag ${ImageName}:latest ${RegistryService}/${ImageName}:latest"
+    echo ${Cmd}
+    eval ${Cmd} || exit
+
+    Cmd="docker tag ${ImageName}:${Version} ${RegistryService}/${ImageName}:${Version}"
+    echo ${Cmd}
+    eval ${Cmd} || exit
+
+    Cmd="docker push ${RegistryService}/${ImageName}:latest"
+    echo ${Cmd}
+    eval ${Cmd} || exit
+
+    Cmd="docker push ${RegistryService}/${ImageName}:${Version}"
+    echo ${Cmd}
+    eval ${Cmd} || exit
+}
+
+

microservices/oe_gateway/Dockerfile

@@ -1,25 +1,19 @@
 FROM python:3.12-slim
 
-ARG ROOT_MCRSVC_DIR=docker_dev/microservices
+COPY requirements.txt /
-ARG FROM_DIR=${ROOT_MCRSVC_DIR}/oe_gateway
 
-COPY ${ROOT_MCRSVC_DIR}/requirements.txt /
 RUN pip install --upgrade pip --root-user-action=ignore
 RUN pip install -r /requirements.txt --root-user-action=ignore
 
 COPY cvttpy /cvttpy
-COPY ${ROOT_MCRSVC_DIR}/.creds  /.creds
 
+RUN mkdir -p /logs /config
 
-COPY ${FROM_DIR}/run_oegw.sh /run_oegw.sh
-RUN chmod +x /run_oegw.sh
-
-RUN mkdir /logs
-RUN mkdir -p /config
-
-
-WORKDIR /
 SHELL ["/bin/bash", "-c"]
-ENV PYTHONPATH=/
-ENTRYPOINT [ "/run_oegw.sh" ]
 
+ENV PYTHONPATH=/
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

microservices/oe_gateway/entrypoint.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+ConfigDir=/config
+LogDir=/logs
+
+Cmd="python3.12"
+Cmd+=" cvttpy/apps/microservices/market_gateways/order_entry_gateway.py"
+Cmd+=" --config=${ConfigDir}/${OEGW_CONFIG_FILE:-cvtt_musvc.cfg}"
+Cmd+=" --credentials_file=${ConfigDir}/${OEGW_CREDS_FILE:-.creds}"
+Cmd+=" --log_file=${LogDir}/${OEGW_LOG_FILE:-%T.oe_gateway.log}"
+Cmd+=" ${OEGW_ADD_ARGS}"
+echo ${Cmd}
+exec ${Cmd}
+

microservices/oe_gateway/requirements.txt

@@ -0,0 +1,11 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5
+hjson>=3.1.0
+sortedcontainers>=2.4.0
+redis>=5.0.8
+python-dateutil>=2.8.2
+types-python-dateutil>=2.8.19.6
+cryptography>=43.0.0
+PyJWT>=2.10.1
+
+

microservices/oe_gateway/run_oegw.sh

@@ -1,48 +0,0 @@
-#!/bin/bash
-
-# runs in container
-
-# runs on host to start container
-usage() {
-    echo -n "Usage: ${0}"
-    echo -n " [-c <config (/config/cvtt.cfg)>]"
-    echo -n " [-a <admin_port (7220)>]"
-    echo -n " [-z (compress log)"]
-    echo
-    exit 1
-}
-
-Config=/config/cvtt.cfg
-AdminPort=7220
-COMPRESS_LOG=false
-
-
-while getopts "c:a:z" opt; do
-    case ${opt} in
-        c ) Config=$OPTARG  ;;
-        a ) AdminPort=$OPTARG ;;
-        z ) COMPRESS_LOG=true ;;
-        \? )
-            echo "Invalid option: -$OPTARG" >&2
-            usage
-            ;;
-        : )
-            echo "Option -$OPTARG requires an argument." >&2
-            usage
-            ;;
-    esac
-done
-
-Cmd="python3.12"
-Cmd="${Cmd} cvttpy/apps/microservices/market_gateways/order_entry_gateway.py"
-Cmd="${Cmd} --config=${Config}"
-Cmd="${Cmd} --credentials_file=/.creds"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_file=/logs/%T.md_gateway.log"
-if ${COMPRESS_LOG} ; then
-    Cmd="${Cmd} --compress_log"
-fi
-echo ${Cmd}
-eval ${Cmd}
-
-

microservices/tester/Dockerfile

@@ -1,26 +0,0 @@
-FROM python:3.12-slim
-
-ARG ROOT_MUSVC_DIR=docker_dev/microservices
-ARG FROM_DIR=${ROOT_MUSVC_DIR}/tester
-
-COPY ${ROOT_MUSVC_DIR}/requirements.txt /
-RUN pip install --upgrade pip --root-user-action=ignore
-RUN pip install -r /requirements.txt --root-user-action=ignore
-
-COPY cvttpy /cvttpy
-# COPY ${ROOT_MUSVC_DIR}/.creds  /.creds
-
-
-COPY ${FROM_DIR}/run_tester.sh /run_tester.sh
-RUN chmod +x /run_tester.sh
-
-RUN mkdir /logs
-RUN mkdir -p /config
-RUN mkdir -p /data
-
-
-WORKDIR /
-SHELL ["/bin/bash", "-c"]
-ENV PYTHONPATH=/
-ENTRYPOINT [ "/run_tester.sh" ]
-

microservices/tester/run_tester.sh

@@ -1,55 +0,0 @@
-#!/bin/bash
-
-# runs in container
-
-# runs on host to start container
-usage() {
-    echo -n "Usage: ${0}"
-    echo -n " [-c <config (/config/cvtt.cfg) >]"
-    echo -n " [-i <instruments (COINBASE:PAIR-BTC-USD,BNBFUT:PERP-BTC-USDT) >]"
-    echo -n " [-d <data types (TRADES) >]"
-    echo -n " [-a <admin_port (7221) >]"
-    echo -n " [-z (compress log (false))"]
-    echo
-    exit 1
-}
-
-Config=/config/cvtt.cfg
-AdminPort=7221
-COMPRESS_LOG=false
-Instruments="COINBASE:PAIR-BTC-USD,BNBFUT:PERP-BTC-USDT"
-DataTypes="TRADES"
-
-while getopts "a:c:d:i:z" opt; do
-    case ${opt} in
-        c ) Config=$OPTARG  ;;
-        a ) AdminPort=$OPTARG ;;
-        i ) Instruments=$OPTARG ;;
-        d ) DataTypes=$OPTARG ;;
-        z ) COMPRESS_LOG=true ;;
-        \? )
-            echo "Invalid option: -$OPTARG" >&2
-            usage
-            ;;
-        : )
-            echo "Option -$OPTARG requires an argument." >&2
-            usage
-            ;;
-    esac
-done
-
-Cmd="python3.12"
-Cmd="${Cmd} cvttpy/apps/microservices/tester/musvc_tester.py"
-Cmd="${Cmd} --config=${Config}"
-Cmd="${Cmd} --credentials_file=/.creds"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --data_types=${DataTypes}"
-Cmd="${Cmd} --instruments=${Instruments}"
-Cmd="${Cmd} --log_file=/logs/%T.tester.log"
-if ${COMPRESS_LOG} ; then
-    Cmd="${Cmd} --compress_log"
-fi
-echo ${Cmd}
-eval ${Cmd}
-
-