VERSION

@@ -1 +1 @@
-0.2.7,new md_recorder build
+0.2.6,improved build.sh for microservices

config_service/Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.12-slim
+
+WORKDIR /
+COPY docker_dev/config_service/requirements.txt /
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+
+# Shared Volumes
+RUN mkdir -p /app/data
+RUN mkdir /logs
+
+ENV PYTHONPATH=/
+CMD [                                        \
+    "python3.12",                            \
+    "cvttpy/apps/utils/config_server.py",    \
+    "--port=6789",                           \
+    "--root=/app/data",                      \
+    "--log_file=/logs/%T.config_service.log" \
+]
+

config_service/HOWTO.md

@@ -0,0 +1,42 @@
+# Build/Dev Host
+## Build
+```bash
+TgtHostname=cloud16.cvtt.vpn
+Version=$(cat cvttpy/VERSION)
+
+cd /home/oleg/develop/cvtt2
+docker build  -t config_service -t config_service:${Version} -f cvttpy/utils/docker/config_service/Dockerfile .
+
+## Deploy
+
+docker save -o /tmp/cvtt_config_service.img.tar config_service
+scp /tmp/cvtt_config_service.img.tar cvtt@${TgtHostname}:/tmp/
+
+ssh cvtt@${TgtHostname}
+```
+
+## On target Machine make sure user can use docker
+```bash
+sudo usermod -a -G docker cvtt
+```
+**Re-login as cvtt after adding cvtt to the group**
+
+```bash
+docker rm -f cvtt_config_service
+docker load -i /tmp/cvtt_config_service.img.tar
+docker run -d --name=cvtt_config_service -p 6789:6789 -v /home/cvtt/prod/config_service/data:/app/data -v /home/cvtt/prod/logs:/logs config_service
+```
+
+## Restarting
+```bash
+docker restart cvtt_config_service
+```
+
+# Useful Commands
+```bash
+docker images
+docker ps -a
+docker image rm <image>
+docker rm <container>
+docker exec -it <container> /bin/bash
+```

config_service/requirements.txt

@@ -0,0 +1,2 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5

market_data/alpaca_hbar/Dockerfile

@@ -24,6 +24,7 @@ COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
 RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub
 
 
+
 # Shared Volumes
 RUN mkdir -p /data
 RUN mkdir /logs

market_data/crypto_md_day/crypto_md_day.sh

@@ -39,8 +39,8 @@ if [ -z "${DbHost}" ] || [ -z "$Date" ]; then
 fi
 
 CredKey=""
-if [ "${DbHost}" == "cloud28" ] ; then
+if [ "${DbHost}" == "cvttdata" ] ; then
-    CredKey=TSDB_MD_CLOUD28_RO
+    CredKey=TSDB_MD_CVTTDATA_RO
 elif [ "${DbHost}" == "cloud21" ] ; then
     CredKey=TSDB_MD_CLD21_RO
 else

market_data/md_recorder/.creds

@@ -0,0 +1,17 @@
+{
+    "__dummy__": null
+    , "TSDB_MD_CLD21": {
+        "host": "cloud21.cvtt.vpn"
+        , "port": 5432
+        , "user": "cvtt"
+        , "database": "cvtt_md"
+        , "password": "ICdIh0JnMM7vM7Pf"
+    }
+    , "TSDB_MD_CVTTDATA": {
+        "host": "cvttdata.cvtt.vpn"
+        , "port": 5432
+        , "user": "cvtt"
+        , "database": "cvtt_md"
+        , "password": "ICdIh0JnMM7vM7Pf"
+    }
+}

market_data/md_recorder/.dockerignore

@@ -0,0 +1,3 @@
+.git
+**/__pycache__
+.pipenv

market_data/md_recorder/Dockerfile

@@ -0,0 +1,27 @@
+FROM python:3.12-slim
+
+ARG FROM_DIR=docker_dev/market_data/md_recorder
+
+RUN apt-get update && apt-get install -y \
+    apt-utils \
+    libpq-dev \
+    build-essential
+
+COPY ${FROM_DIR}/requirements.txt /
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+COPY ${FROM_DIR}/run_md_recorder.sh /run_md_recorder.sh
+
+
+# Shared Volumes
+RUN mkdir -p /app/data
+RUN mkdir /logs
+RUN chmod +x /run_md_recorder.sh
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENV PYTHONPATH=/
+ENTRYPOINT [ "/run_md_recorder.sh" ]
+

market_data/md_recorder/run_md_recorder.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# runs in container
+
+ValidJobs=('BNBFUT_CLD21' 'BNBSPOT_CLD21' 'COINBASE_CLD21' 'BNBFUT_CVTTDATA' 'BNBSPOT_CVTTDATA' 'COINBASE_CVTTDATA')
+# runs on host to start container
+usage() {
+  echo "Usage: $0 <job_name; one of (${ValidJobs[@]})>"
+  exit 1
+}
+
+is_valid() {
+  local job=$1
+  for valid_job in "${ValidJobs[@]}";
+  do
+    # echo "job=$job valid_job=$valid_job"
+    if [[ "${job}" == "${valid_job}" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+job=${1}
+if  ! is_valid "${job}"; then
+    usage
+fi
+
+ConfigServer=cloud16.cvtt.vpn:6789
+
+Cmd="python3.12"
+Cmd="${Cmd} cvttpy/apps/md/md_recorder.py"
+Cmd="${Cmd} --config=http://${ConfigServer}/apps/md_recorder"
+Cmd="${Cmd} --credentials_file=/.creds"
+Cmd="${Cmd} --compress_log"
+Cmd="${Cmd} --instrument_group=${job}"
+Cmd="${Cmd} --log_file=/logs/%T.MD_REC.${job}.log"
+if [ "${job}" == "BNBFUT_CLD21" ] ; then
+    Cmd="${Cmd} --exch_acct_cfgname=BNBFUT"
+    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
+    Cmd="${Cmd} --admin_port=7201"
+elif [ "${job}" == "COINBASE_CLD21" ] ; then
+    Cmd="${Cmd} --exch_acct_cfgname=COINBASE_AT"
+    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
+    Cmd="${Cmd} --admin_port=7202"
+elif [ "${job}" == "BNBSPOT_CLD21" ] ; then
+    Cmd="${Cmd} --exch_acct_cfgname=BNBSPOT"
+    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
+    Cmd="${Cmd} --admin_port=7203"
+elif [ "${job}" == "BNBSPOT_CVTTDATA" ] ; then
+    Cmd="${Cmd} --exch_acct_cfgname=BNBSPOT"
+    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
+    Cmd="${Cmd} --admin_port=7204"
+elif [ "${job}" == "BNBFUT_CVTTDATA" ] ; then
+    Cmd="${Cmd} --exch_acct_cfgname=BNBFUT"
+    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
+    Cmd="${Cmd} --admin_port=7205"
+elif [ "${job}" == "COINBASE_CVTTDATA" ] ; then
+    Cmd="${Cmd} --exch_acct_cfgname=COINBASE_AT"
+    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
+    Cmd="${Cmd} --admin_port=7206"
+else
+    echo "Unrecognized JOB: ${job}"
+    exit 1
+fi
+echo ${Cmd}
+eval ${Cmd}
+
+

market_data/md_recorder_monitor/.dockerignore

@@ -0,0 +1,3 @@
+.git
+**/__pycache__
+.pipenv

market_data/md_recorder_monitor/requirements.txt

@@ -0,0 +1,7 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5
+psycopg>=3.2.1
+hjson>=3.1.0
+pandas>=1.5.3
+sortedcontainers>=2.4.0
+redis>=5.0.8

microservices/.env

@@ -0,0 +1,18 @@
+CVTT_USER=1000:1000
+CVTT_VERSION=0.0.1
+
+# MDGW_CONFIG_FILE=config_musvc.cfg
+# MDGW_CREDS_FILE=.creds
+MDGW_LOG_FILE=%D.md_gateway.log
+MDGW_ADD_ARGS=
+
+# OEGW_CONFIG_FILE=config_musvc.cfg
+# OEGW_CREDS_FILE=.creds
+OEGW_LOG_FILE=%D.oe_gateway.log
+#OEGW_ADD_ARGS="--log_level=DEBUG --debug_mode"
+OEGW_ADD_ARGS="--debug_mode"
+
+# ACGW_CONFIG_FILE=config_musvc.cfg
+# ACGW_CREDS_FILE=.creds
+ACGW_LOG_FILE=%D.ac_gateway.log
+ACGW_ADD_ARGS=

microservices/ac_gateway/.dockerignore

@@ -0,0 +1,3 @@
+.git
+**/__pycache__
+.pipenv

microservices/ac_gateway/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.12-slim
+
+COPY requirements.txt /
+
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+
+RUN mkdir -p /logs /config
+
+SHELL ["/bin/bash", "-c"]
+
+ENV PYTHONPATH=/
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

microservices/ac_gateway/entrypoint.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+ConfigDir=/config
+LogDir=/logs
+
+Cmd="python3.12"
+Cmd+=" cvttpy/apps/microservices/market_gateways/accounting_gateway.py"
+Cmd+=" --config=${ConfigDir}/${ACGW_CONFIG_FILE:-cvtt_musvc.cfg}"
+Cmd+=" --credentials_file=${ConfigDir}/${ACGW_CREDS_FILE:-.creds}"
+Cmd+=" --log_file=${LogDir}/${ACGW_LOG_FILE:-%T.ac_gateway.log}"
+Cmd+=" ${ACGW_ADD_ARGS}"
+echo ${Cmd}
+exec ${Cmd}
+

microservices/ac_gateway/requirements.txt

@@ -0,0 +1,9 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5
+hjson>=3.1.0
+sortedcontainers>=2.4.0
+redis>=5.0.8
+python-dateutil>=2.8.2
+types-python-dateutil>=2.8.19.6
+cryptography>=43.0.0
+PyJWT>=2.10.1

microservices/build.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+
+usage() {
+  echo "Usage: $0 -I <image_name> -V <image_version> -L <library>:<library_version>"
+  exit 1
+}
+
+# --- Settings
+RegistryService=cloud21.cvtt.vpn:5500
+# --- Settings
+
+
+# ---------------- cmdline
+while getopts "I:V:L:" opt; do
+  case ${opt} in
+    I )
+      ImageName=$OPTARG
+      ;;
+    V )
+      ImageVersion=$OPTARG
+      ;;
+    L )
+      Library=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+# ---------------- cmdline
+
+if [ "${ImageName}" == "" ]
+then
+    usage
+fi
+
+if [ "${ImageVersion}" == "" ]
+then
+    usage
+fi
+
+Project=
+ProjectVersion=
+if  [ "${Library}" != "" ] ; then
+    IFS=':' read -ra parts <<< "${Library}"
+    Project=${parts[0]}
+    ProjectVersion=${parts[1]}
+fi
+# exit
+
+DockerDir=$(realpath $(dirname ${0})/${ImageName})
+
+cd ${DockerDir}
+
+source ../mu_svc_functions.sh
+if [ "${Project}" != "" ]; then
+    get_project_version ${Project} ${ProjectVersion}
+fi
+
+
+function cleanup {
+    cd ${DockerDir}
+    rm -rf cvttpy
+}
+trap cleanup EXIT
+
+build_docker_image ${ImageName} ${RegistryService} ${ImageVersion}
+
+
+# echo "**** D E B U G  E X I T" && exit
+echo "***** ${0}  D O N E"
+
+

microservices/docker-compose.yml

@@ -0,0 +1,58 @@
+# CVTT Microservices
+services:
+    md_gateway:
+        image: cloud21.cvtt.vpn:5500/md_gateway:${CVTT_VERSION}
+        container_name: md_gateway
+        user: ${CVTT_USER:-1001:1001}
+        environment:
+          CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          MDGW_CONFIG_FILE:  ${MDGW_CONFIG_FILE:-cvtt_musvc.cfg}
+          MDGW_CREDS_FILE:  ${MDGW_CONFIG_FILE:-.creds}
+          MDGW_LOG_FILE:  ${MDGW_LOG_FILE:-%T.md_gateway.log}
+          MDGW_ADD_ARGS: ${MDGW_ADD_ARGS:-}
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+        depends_on:
+          - redis
+
+    oe_gateway:
+        image: cloud21.cvtt.vpn:5500/oe_gateway:${CVTT_VERSION}
+        container_name: oe_gateway
+        user: ${CVTT_USER:-1001:1001}
+        environment:
+          CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          OEGW_CONFIG_FILE:  ${OEGW_CONFIG_FILE:-cvtt_musvc.cfg}
+          OEGW_CREDS_FILE:  ${OEGW_CONFIG_FILE:-.creds}
+          OEGW_LOG_FILE:  ${OEGW_LOG_FILE:-%T.oe_gateway.log}
+          OEGW_ADD_ARGS: ${OEGW_ADD_ARGS:-}
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+        depends_on:
+          - redis
+
+    ac_gateway:
+        image: cloud21.cvtt.vpn:5500/ac_gateway:${CVTT_VERSION}
+        container_name: ac_gateway
+        user: ${CVTT_USER:-1001:1001}
+        environment:
+          CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          ACGW_CONFIG_FILE:  ${ACGW_CONFIG_FILE:-cvtt_musvc.cfg}
+          ACGW_CREDS_FILE:  ${ACGW_CONFIG_FILE:-.creds}
+          ACGW_LOG_FILE:  ${ACGW_LOG_FILE:-%T.ac_gateway.log}
+          ACGW_ADD_ARGS: ${ACGW_ADD_ARGS:-}
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+        depends_on:
+          - redis
+
+    redis:
+        image: redis:latest
+        container_name: redis-cvtt
+        ports:
+          - "16379:6379"
+        volumes:
+          - ./data/redis:/data
+

microservices/md_gateway/.dockerignore

@@ -0,0 +1,3 @@
+.git
+**/__pycache__
+.pipenv

microservices/md_gateway/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.12-slim
+
+COPY requirements.txt /
+
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+
+RUN mkdir -p /logs /config
+
+SHELL ["/bin/bash", "-c"]
+
+ENV PYTHONPATH=/
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

microservices/md_gateway/entrypoint.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+ConfigDir=/config
+LogDir=/logs
+
+Cmd="python3.12"
+Cmd+=" cvttpy/apps/microservices/market_gateways/md_gateway.py"
+Cmd+=" --config=${ConfigDir}/${MDGW_CONFIG_FILE:-cvtt_musvc.cfg}"
+Cmd+=" --credentials_file=${ConfigDir}/${MDGW_CREDS_FILE:-.creds}"
+Cmd+=" --log_file=${LogDir}/${MDGW_LOG_FILE:-%T.md_gateway.log}"
+Cmd+=" ${MDGW_ADD_ARGS}"
+echo ${Cmd}
+exec ${Cmd}
+

microservices/md_gateway/requirements.txt

@@ -0,0 +1,9 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5
+hjson>=3.1.0
+sortedcontainers>=2.4.0
+redis>=5.0.8
+python-dateutil>=2.8.2
+types-python-dateutil>=2.8.19.6
+cryptography>=43.0.0
+PyJWT>=2.10.1

microservices/mu_svc_functions.sh

@@ -17,7 +17,7 @@ get_project_version() {
     echo "Checking ${user}@${host} for ${rel_dir}/${Project}/${Version} ..."
     if ssh -q -p ${port} ${user}@${host} "test -d ${rel_dir}/${Project}/${Version}"
     then
-        echo "Getting Version \"${Version}\"..."
+        echo "Version ${Version} found..."
         rsync_cmd="rsync -ahv -e \"ssh -p ${port}\""
         rsync_cmd="${rsync_cmd} ${user}@${host}:${rel_dir}/${Project}/${Version}/"
         rsync_cmd="${rsync_cmd} ./"
@@ -41,13 +41,8 @@ build_docker_image() {
     ImageName=${1}
     RegistryService=${2}
     Version=${3}
-    Project=${4}
-    ProjectVersion=${5}
 
     Cmd="docker build"
-    if [ "${ProjectVersion}" != "" ]; then
-        Cmd+=" --label ${Project}=\"${ProjectVersion}\""
-    fi
     Cmd+=" -t ${ImageName}"
     Cmd+=" -t ${ImageName}:latest"
     Cmd+=" -t ${ImageName}:${Version}"

microservices/oe_gateway/.dockerignore

@@ -0,0 +1,3 @@
+.git
+**/__pycache__
+.pipenv

microservices/oe_gateway/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.12-slim
+
+COPY requirements.txt /
+
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+
+RUN mkdir -p /logs /config
+
+SHELL ["/bin/bash", "-c"]
+
+ENV PYTHONPATH=/
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

microservices/oe_gateway/entrypoint.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+ConfigDir=/config
+LogDir=/logs
+
+Cmd="python3.12"
+Cmd+=" cvttpy/apps/microservices/market_gateways/order_entry_gateway.py"
+Cmd+=" --config=${ConfigDir}/${OEGW_CONFIG_FILE:-cvtt_musvc.cfg}"
+Cmd+=" --credentials_file=${ConfigDir}/${OEGW_CREDS_FILE:-.creds}"
+Cmd+=" --log_file=${LogDir}/${OEGW_LOG_FILE:-%T.oe_gateway.log}"
+Cmd+=" ${OEGW_ADD_ARGS}"
+echo ${Cmd}
+exec ${Cmd}
+

microservices/oe_gateway/requirements.txt

@@ -0,0 +1,11 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5
+hjson>=3.1.0
+sortedcontainers>=2.4.0
+redis>=5.0.8
+python-dateutil>=2.8.2
+types-python-dateutil>=2.8.19.6
+cryptography>=43.0.0
+PyJWT>=2.10.1
+
+

test/rsync_test/Dockerfile

@@ -0,0 +1,21 @@
+FROM python:3.12-slim
+
+ARG FROM_DIR=docker_dev/test/rsync_test
+
+COPY ${FROM_DIR}/test.sh /test.sh
+
+RUN apt-get update && apt-get install -y rsync openssh-client
+
+COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
+COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
+
+RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub
+
+# Shared Volumes
+RUN chmod +x /test.sh
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENV PYTHONPATH=/
+ENTRYPOINT [ "/test.sh" ]
+

test/rsync_test/test.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+date > /testfile.txt
+
+Source=/testfile.txt
+Targets=
+Targets="${Targets} cvtt@cloud21.cvtt.vpn:/tmp/"
+Targets="${Targets} cvtt@hs01.cvtt.vpn:/tmp/"
+
+
+for tgt in ${Targets}
+do
+    Cmd="/usr/bin/rsync -ahv"
+    Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
+    Cmd+=" ${Source} ${tgt}"
+    echo $Cmd
+    eval $Cmd
+done
+