progress (UNTESTED)

VERSION

@@ -0,0 +1 @@
+0.2.7,new md_recorder build

__TBD__/buildx.sh

@@ -1,74 +0,0 @@
-#!/bin/bash
-
-echo "Script $0 does not work yet. Aborted" && exit
-
-function usage {
-    echo "Usage: ${0} <app_path (e.g. market_data/md_recorder_monitor)> <version>"
-    exit
-}
-
-AppPath=${1}
-CvttpyVersion=${2}
-
-if [ "${AppPath}" == "" ]
-then
-    usage
-fi
-
-# --- Settings
-DockerDir=$(realpath $(dirname ${0})/${AppPath})
-RootDir=$(realpath $(dirname ${0})/..)
-RegistryService=cloud21.cvtt.vpn:5500
-
-AppName=$(basename ${AppPath})
-
-ImageName=${AppName}
-ImageDir=${HOME}/docker_images
-mkdir -p ${ImageDir}
-
-Version=$(cat ${RootDir}/cvttpy/release_version.txt | awk -F, '{print $1}')
-
-cd ${RootDir}
-
-# Create a new builder instance
-Cmd="docker buildx create --name ${AppName}_builder"
-echo ${Cmd}
-eval ${Cmd} || exit
-
-# Switch to the newly created builder
-Cmd="docker buildx use ${AppName}_builder"
-echo ${Cmd}
-eval ${Cmd}
-
-# Ensure buildx is bootstrapped
-Cmd="docker buildx inspect ${AppName}_builder --bootstrap"
-echo ${Cmd}
-eval ${Cmd}
-
-# Build the image using buildx, tagging with both 'latest' and the version number, and pushing directly
-Cmd="docker buildx build --platform linux/amd64,linux/arm64 -t ${ImageName}:latest -t ${ImageName}:${Version} -f ${DockerDir}/Dockerfile . --push"
-echo ${Cmd}
-eval ${Cmd}
-
-# Tagging the images with the registry
-Cmd="docker tag ${ImageName}:latest ${RegistryService}/${ImageName}:latest"
-echo ${Cmd}
-eval ${Cmd}
-
-Cmd="docker tag ${ImageName}:${Version} ${RegistryService}/${ImageName}:${Version}"
-echo ${Cmd}
-eval ${Cmd}
-
-# Push the images to the registry
-Cmd="docker push ${RegistryService}/${ImageName}:latest"
-echo ${Cmd}
-eval ${Cmd}
-
-Cmd="docker push ${RegistryService}/${ImageName}:${Version}"
-echo ${Cmd}
-eval ${Cmd}
-
-# Optionally, remove the builder to clean up
-Cmd="docker buildx rm ${AppName}_builder"
-echo ${Cmd}
-eval ${Cmd}

build.sh

@@ -25,7 +25,7 @@ ImageDir=${HOME}/docker_images
 mkdir -p ${ImageDir}
 
 
-Version=$(cat ${RootDir}/${Project}/release_version.txt | awk -F, '{print $1}')
+Version=$(cat ${RootDir}/${Project}/VERSION | awk -F, '{print $1}')
 # Version=${ProjectVersion}
 
 cd ${RootDir}

config_service/HOWTO.md

@@ -2,7 +2,7 @@
 ## Build
 ```bash
 TgtHostname=cloud16.cvtt.vpn
-Version=$(cat cvttpy/release_version.txt)
+Version=$(cat cvttpy/VERSION)
 
 cd /home/oleg/develop/cvtt2
 docker build  -t config_service -t config_service:${Version} -f cvttpy/utils/docker/config_service/Dockerfile .

dckr_bld_functions.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+get_project_version() {
+    Project=${1}
+    Version=${2}
+
+    host="cloud21.cvtt.vpn"
+    port="22"
+    user="cvttdist"
+    rel_dir="/home/cvttdist/software/cvtt2"
+
+    if [ "${Version}" == "latest" ]; then
+        echo "Checking for latest version of ${Project} on ${user}@${host}:${rel_dir}"
+        Version=$(ssh -q -p ${port} ${user}@${host} "ls -tr ${rel_dir}/${Project} | tail -1" )
+        echo "Latest version is ${Version}"
+    fi
+    echo "Checking ${user}@${host} for ${rel_dir}/${Project}/${Version} ..."
+    if ssh -q -p ${port} ${user}@${host} "test -d ${rel_dir}/${Project}/${Version}"
+    then
+        echo "Getting Version \"${Version}\"..."
+        rsync_cmd="rsync -ahv -e \"ssh -p ${port}\""
+        rsync_cmd="${rsync_cmd} ${user}@${host}:${rel_dir}/${Project}/${Version}/"
+        rsync_cmd="${rsync_cmd} ./"
+        echo ${rsync_cmd}
+        eval ${rsync_cmd}
+        status=$?
+        if [ ${status} -eq 0 ]
+        then
+            echo "Loading successful..."
+            return 0
+        else
+            return 1
+        fi
+    else
+        echo "Not Found ${rel_dir}/${Project}/${Version} on ${user}@${host}"
+        return 1
+    fi
+}
+
+build_docker_image() {
+    ImageName=${1}
+    RegistryService=${2}
+    Version=${3}
+    Project=${4}
+    ProjectVersion=${5}
+
+    Cmd="docker build"
+    if [ "${ProjectVersion}" != "" ]; then
+        Cmd+=" --label ${Project}=\"${ProjectVersion}\""
+    fi
+    Cmd+=" -t ${ImageName}"
+    Cmd+=" -t ${ImageName}:latest"
+    Cmd+=" -t ${ImageName}:${Version}"
+    Cmd+=" -f Dockerfile"
+    Cmd+=" ."
+    echo ${Cmd}
+    eval ${Cmd} || exit
+
+    Cmd="docker tag ${ImageName}:latest ${RegistryService}/${ImageName}:latest"
+    echo ${Cmd}
+    eval ${Cmd} || exit
+
+    Cmd="docker tag ${ImageName}:${Version} ${RegistryService}/${ImageName}:${Version}"
+    echo ${Cmd}
+    eval ${Cmd} || exit
+
+    Cmd="docker push ${RegistryService}/${ImageName}:latest"
+    echo ${Cmd}
+    eval ${Cmd} || exit
+
+    Cmd="docker push ${RegistryService}/${ImageName}:${Version}"
+    echo ${Cmd}
+    eval ${Cmd} || exit
+}
+
+

market_data/alpaca_hbar/Dockerfile

@@ -20,9 +20,9 @@ COPY ${FROM_DIR}/alpaca_hbar.sh /alpaca_hbar.sh
 
 COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
 COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
-COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
 
-RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
+RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub
+
 
 
 # Shared Volumes

market_data/alpaca_hbar/alpaca_hbar.sh

@@ -50,7 +50,7 @@ mkdir -p ${LogDir}
 
 echo "CommandLine: ${*}"
 
-export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+export CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS
 
 
 if [ -z ${Start}  ] ; then
@@ -99,7 +99,9 @@ Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
 
 for tgt in ${Targets}
 do
-    Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
+    Cmd="/usr/bin/rsync -ahv"
+    Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
+    Cmd+=" ${Source} ${tgt}"
     echo $Cmd
     eval $Cmd
 done

market_data/alpaca_qat/Dockerfile

@@ -21,9 +21,8 @@ COPY ${FROM_DIR}/alpaca_qat.sh /alpaca_qat.sh
 
 COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
 COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
-COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
 
-RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
+RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub
 
 
 # Shared Volumes

market_data/alpaca_qat/alpaca_qat.sh

@@ -54,7 +54,7 @@ mkdir -p ${LogDir}
 
 echo "CommandLine: ${*}"
 
-export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+export CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS
 
 
 if [ -z ${Start}  ] ; then
@@ -101,7 +101,9 @@ Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
 
 for tgt in ${Targets}
 do
-    Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
+    Cmd="/usr/bin/rsync -ahv"
+    Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
+    Cmd+=" ${Source} ${tgt}"
     echo $Cmd
     eval $Cmd
 done

market_data/crypto_md_day/Dockerfile

@@ -6,7 +6,6 @@ COPY ${FROM_DIR}/requirements.txt /
 
 
 COPY cvttpy /cvttpy
-# COPY ${FROM_DIR}/.creds  /.creds
 COPY ${FROM_DIR}/crypto_md_day.sh /crypto_md_day.sh
 
 RUN pip install --upgrade pip --root-user-action=ignore
@@ -19,9 +18,8 @@ RUN apt-get update && apt-get install -y rsync openssh-client
 
 COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
 COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
-COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
 
-RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
+RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub
 
 
 # Shared Volumes

market_data/crypto_md_day/HOWTO.md

@@ -1,7 +1,7 @@
 # Build/Dev Host
 ## Build
 ```bash
-Version=$(cat /home/oleg/develop/cvtt2/cvttpy/release_version.txt)
+Version=$(cat /home/oleg/develop/cvtt2/cvttpy/VERSION)
 
 cd /home/oleg/develop/cvtt2
 docker build  -t crypto_md_day -t crypto_md_day:${Version} -f docker_dev/crypto_md_day/Dockerfile .

market_data/crypto_md_day/crypto_md_day.sh

@@ -51,16 +51,16 @@ fi
 export PYTHONPATH=/
 
 Cmd="python3.12"
-Cmd="${Cmd} cvttpy/research/utils/archive_ts_md.py"
-Cmd="${Cmd} --config=http://cloud23.cvtt.vpn:6789/apps/md_recorder"
-Cmd="${Cmd} --db_credentials_key=${CredKey}"
-Cmd="${Cmd} --credentials_file=/.creds"
-Cmd="${Cmd} --date=${Date}"
-Cmd="${Cmd} --schemas=${Schemas}"
-Cmd="${Cmd} --root_dir=/app/data/${DbHost}"
-Cmd="${Cmd} --format=SQLite"
-Cmd="${Cmd} --compress"
-# Cmd="${Cmd} --log_file=/logs/%T.crypto_md_day.log"
+Cmd+=" cvttpy/research/utils/archive_ts_md.py"
+Cmd+=" --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder"
+Cmd+=" --db_credentials_key=${CredKey}"
+Cmd+=" --credentials_file=/.creds"
+Cmd+=" --date=${Date}"
+Cmd+=" --schemas=${Schemas}"
+Cmd+=" --root_dir=/app/data/${DbHost}"
+Cmd+=" --format=SQLite"
+Cmd+=" --compress"
+# Cmd+=" --log_file=/logs/%T.crypto_md_day.log"
 
 echo ${Cmd}
 eval ${Cmd}
@@ -72,10 +72,11 @@ Targets=
 Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto"
 Targets="${Targets} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto"
 
-
 for tgt in ${Targets}
 do
-    Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
+    Cmd="/usr/bin/rsync -ahv"
+    Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
+    Cmd+=" ${Source} ${tgt}"
     echo $Cmd
     eval $Cmd
 done

market_data/md_recorder/Dockerfile

@@ -1,27 +1,25 @@
 FROM python:3.12-slim
 
-ARG FROM_DIR=docker_dev/market_data/md_recorder
-
+# libpq needed for Timescale
 RUN apt-get update && apt-get install -y \
     apt-utils \
     libpq-dev \
     build-essential
 
-COPY ${FROM_DIR}/requirements.txt /
+COPY requirements.txt /
+
 RUN pip install --upgrade pip --root-user-action=ignore
 RUN pip install -r /requirements.txt --root-user-action=ignore
 
 COPY cvttpy /cvttpy
-COPY ${FROM_DIR}/run_md_recorder.sh /run_md_recorder.sh
 
+RUN mkdir -p /logs /config
 
-# Shared Volumes
-RUN mkdir -p /app/data
-RUN mkdir /logs
-RUN chmod +x /run_md_recorder.sh
-
-WORKDIR /
 SHELL ["/bin/bash", "-c"]
-ENV PYTHONPATH=/
-ENTRYPOINT [ "/run_md_recorder.sh" ]
 
+ENV PYTHONPATH=/
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

market_data/md_recorder/build.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# --- Settings
+RegistryService=cloud21.cvtt.vpn:5500
+ImageName=md_recorder
+BashFunctionCode=$(realpath $(dirname ${0}))/../../dckr_bld_functions.sh
+# --- Settings
+
+usage() {
+  echo "Usage: $0 -V <image_version> -L <library>:<library_version>"
+  exit 1
+}
+
+
+# ---------------- cmdline
+while getopts "V:L:" opt; do
+  case ${opt} in
+    V )
+      ImageVersion=$OPTARG
+      ;;
+    L )
+      Library=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+# ---------------- cmdline
+
+if [ "${ImageVersion}" == "" ]
+then
+    usage
+fi
+
+Project=
+ProjectVersion=
+if  [ "${Library}" != "" ] ; then
+    IFS=':' read -ra parts <<< "${Library}"
+    Project=${parts[0]}
+    ProjectVersion=${parts[1]}
+fi
+
+DockerDir=$(realpath $(dirname ${0}))
+
+cd ${DockerDir}
+source ${BashFunctionCode}
+
+if [ "${Project}" != "" ]; then
+    get_project_version ${Project} ${ProjectVersion}
+fi
+function cleanup {
+    cd ${DockerDir}
+    rm -rf cvttpy
+}
+trap cleanup EXIT
+
+build_docker_image ${ImageName} ${RegistryService} ${ImageVersion} ${Project} ${ProjectVersion}
+
+
+# echo "**** D E B U G  E X I T" && exit
+echo "***** ${0}  D O N E"
+
+

market_data/md_recorder/deployment/cloud24/.env

@@ -0,0 +1,52 @@
+# ----- cloud24 settings
+CVTT_USER=1001:1001
+IMAGE_VERSION=0.0.2
+
+CONFIG_FILE=/config/md_recorder.cfg
+CREDS_FILE=/config/.creds
+
+# bnspot-cvttdata
+BSCD_CONTAINER_NAME=md_recorder.BNBSPOT_CVTTDATA
+BSCD_INSTRUMENT_GROUP=BNBSPOT_CVTTDATA
+BSCD_EXCHANGE_ACCT_NAME=BNBSPOT
+BSCD_DB_CRED_KEY=TSDB_MD_CVTTDATA
+
+BSCD_LOG_FILE=/logs/%T.MD_REC.${BSCD_INSTRUMENT_GROUP}.log
+BSCD_ADD_ARGS=--compress_log
+
+BSCD_ADMIN_PORT=7201
+
+# bnspot-cloud21
+BSCLD21_CONTAINER_NAME=md_recorder.BNBSPOT_CLD21
+BSCLD21_INSTRUMENT_GROUP=BNBSPOT_CLD21
+BSCLD21_EXCHANGE_ACCT_NAME=BNBSPOT
+BSCLD21_DB_CRED_KEY=TSDB_MD_CLD21
+
+BSCLD21_LOG_FILE=/logs/%T.MD_REC.${BSCLD21_INSTRUMENT_GROUP}.log
+BSCLD21_ADD_ARGS=--compress_log
+
+BSCLD21_ADMIN_PORT=7202
+
+# bnbfut-cloud21
+BFCLD21_CONTAINER_NAME=md_recorder.BNBFUT_CLD21
+BFCLD21_INSTRUMENT_GROUP=BNBFUT_CLD21
+BFCLD21_EXCHANGE_ACCT_NAME=BNBFUT
+BFCLD21_DB_CRED_KEY=TSDB_MD_CLD21
+
+BFCLD21_LOG_FILE=/logs/%T.MD_REC.${BFCLD21_INSTRUMENT_GROUP}.log
+BFCLD21_ADD_ARGS=--compress_log
+
+BFCLD21_ADMIN_PORT=7203
+
+# coinbase-cloud21
+CBCLD21_CONTAINER_NAME=md_recorder.COINBASE_CLD21
+CBCLD21_INSTRUMENT_GROUP=COINBASE_CLD21
+CBCLD21_EXCHANGE_ACCT_NAME=COINBASE_AT
+CBCLD21_DB_CRED_KEY=TSDB_MD_CLD21
+
+CBCLD21_LOG_FILE=/logs/%T.MD_REC.${CBCLD21_INSTRUMENT_GROUP}.log
+CBCLD21_ADD_ARGS=--compress_log
+
+CBCLD21_ADMIN_PORT=7204
+
+

market_data/md_recorder/deployment/cloud24/docker-compose.yml

@@ -0,0 +1,86 @@
+# CVTT Microservices
+services:
+    bfcld21_md_recorder:
+        image: cloud21.cvtt.vpn:5500/md_recorder:${IMAGE_VERSION}
+        container_name: ${BFCLD21_CONTAINER_NAME}
+        user: ${CVTT_USER:-1001:1001}
+        pull_policy: always
+        restart: unless-stopped
+        environment:
+          CONFIG_SERVICE:     ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          CONFIG_FILE:        ${CONFIG_FILE:-/config/md_recorder.cfg}
+          CREDS_FILE:         ${CREDS_FILE}
+          EXCHANGE_ACCT_NAME: ${BFCLD21_EXCHANGE_ACCT_NAME}
+          INSTRUMENT_GROUP:   ${BFCLD21_INSTRUMENT_GROUP}
+          DB_CRED_KEY:        ${BFCLD21_DB_CRED_KEY}
+          LOG_FILE:           ${BFCLD21_LOG_FILE}
+          ADD_ARGS:           ${BFCLD21_ADD_ARGS:-}
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+        ports:
+          - ${BFCLD21_ADMIN_PORT}:7200
+
+
+    cbcld21_md_recorder:
+        image: cloud21.cvtt.vpn:5500/md_recorder:${IMAGE_VERSION}
+        container_name: ${CBCLD21_CONTAINER_NAME}
+        user: ${CVTT_USER:-1001:1001}
+        pull_policy: always
+        restart: unless-stopped
+        environment:
+          CONFIG_SERVICE:     ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          CONFIG_FILE:        ${CONFIG_FILE:-/config/md_recorder.cfg}
+          CREDS_FILE:         ${CREDS_FILE}
+          EXCHANGE_ACCT_NAME: ${CBCLD21_EXCHANGE_ACCT_NAME}
+          INSTRUMENT_GROUP:   ${CBCLD21_INSTRUMENT_GROUP}
+          DB_CRED_KEY:        ${CBCLD21_DB_CRED_KEY}
+          LOG_FILE:           ${CBCLD21_LOG_FILE}
+          ADD_ARGS:           ${CBCLD21_ADD_ARGS:-}
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+        ports:
+          - ${CBCLD21_ADMIN_PORT}:7200
+
+    bscld21_md_recorder:
+        image: cloud21.cvtt.vpn:5500/md_recorder:${IMAGE_VERSION}
+        container_name: ${BSCLD21_CONTAINER_NAME}
+        user: ${CVTT_USER:-1001:1001}
+        pull_policy: always
+        restart: unless-stopped
+        environment:
+          CONFIG_SERVICE:     ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          CONFIG_FILE:        ${CONFIG_FILE:-/config/md_recorder.cfg}
+          CREDS_FILE:         ${CREDS_FILE}
+          EXCHANGE_ACCT_NAME: ${BSCLD21_EXCHANGE_ACCT_NAME}
+          INSTRUMENT_GROUP:   ${BSCLD21_INSTRUMENT_GROUP}
+          DB_CRED_KEY:        ${BSCLD21_DB_CRED_KEY}
+          LOG_FILE:           ${BSCLD21_LOG_FILE}
+          ADD_ARGS:           ${BSCLD21_ADD_ARGS:-}
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+        ports:
+          - ${BSCLD21_ADMIN_PORT}:7200
+
+    bscd_md_recorder:
+        image: cloud21.cvtt.vpn:5500/md_recorder:${IMAGE_VERSION}
+        container_name: ${BSCD_CONTAINER_NAME}
+        user: ${CVTT_USER:-1001:1001}
+        pull_policy: always
+        restart: unless-stopped
+        environment:
+          CONFIG_SERVICE:     ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          CONFIG_FILE:        ${CONFIG_FILE:-/config/md_recorder.cfg}
+          CREDS_FILE:         ${CREDS_FILE}
+          EXCHANGE_ACCT_NAME: ${BSCD_EXCHANGE_ACCT_NAME}
+          INSTRUMENT_GROUP:   ${BSCD_INSTRUMENT_GROUP}
+          DB_CRED_KEY:        ${BSCD_DB_CRED_KEY}
+          LOG_FILE:           ${BSCD_LOG_FILE}
+          ADD_ARGS:           ${BSCD_ADD_ARGS:-}
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+        ports:
+          - ${BSCD_ADMIN_PORT}:7200

market_data/md_recorder/deployment/cloud24/md_recorder.cfg

@@ -0,0 +1,206 @@
+{
+    "refdata": {
+        "assets": @inc=http://@env{CONFIG_SERVICE}/refdata/assets
+        , "instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/instruments
+        , "exchange_instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/exchange_instruments
+        , "dynamic_instrument_exchanges": ["ALPACA"]
+        , "exchanges": {
+            "COINBASE_AT": {
+                "exchange_id":          "COINBASE"
+                , "name":               "Coinbase Advanced Trade"
+                , "type":               "SPOT"
+
+                , "mktdata" : {
+                    # ****** NEW VAlUE
+                    # "class":  "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMultiWsFeed"
+                    "class":  "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMdFeed"
+                    # ****** NEW VAlUE
+
+                    , "reconnect_secs": 2
+                    , "max_connections": 10
+                    , "price_throttle": true
+
+                    , "ws_url": ["WSS", "wss://ws-feed.exchange.coinbase.com"]
+                }
+            }
+            , "BNBSPOT": {
+                "exchange_id":          "BNBSPOT"
+                , "name":               "Binance Spot"
+                , "type":               "SPOT"
+
+                , "mktdata" : {
+                    # -------- NEW VALUE
+                    # "class":           "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMultiWsFeed"
+                    "class": "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMktDataFeed"
+                    # -------- NEW VALUE
+                    , "max_connections": 10
+                    , "reconnect_secs": 2.0
+
+
+                    , "ws_url": ["WSS", "wss://stream.binance.com/ws"]
+                    , "REST": {
+                        "depth_snapshot_url": ["GET", "https://api.binance.com/api/v3/depth"],
+                    }
+                }
+            }
+            , "BNBFUT": {
+                "exchange_id":          "BNBFUT"
+                , "name":               "Binance Futures USD-M"
+                , "type":               "FUTURES"
+
+                , "mktdata" : {
+                    # -------- NEW VALUE
+                    # "class":           "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMultiWsFeed"
+                    "class":           "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMktDataFeed"
+                    # -------- NEW VALUE
+                    , "max_connections": 10
+                    , "reconnect_secs": 2.0
+
+                    , "ws_url": ["WSS", "wss://fstream.binance.com/ws"]
+
+                    , "REST": {
+                        # https://fapi.binance.com/fapi/v1/depth?symbol=BNBBTC&limit=1000
+                        "depth_snapshot_url": ["GET", "https://fapi.binance.com/fapi/v1/depth"],
+                    }
+                }
+            }
+        }
+    }
+    , "comm": @inc=http://@env{CONFIG_SERVICE}/apps/common/comm
+
+    , "md_recorder": {
+        "instrument_groups": {
+            "BNBFUT_CLD21": [
+                "BNBFUT:PERP-BTC-USDT"
+                , "BNBFUT:PERP-ETH-USDT"
+            ]
+            "BNBFUT_CVTTDATA": [
+                "BNBFUT:PERP-BTC-USDT"
+                , "BNBFUT:PERP-ETH-USDT"
+            ]
+            , "BNBSPOT_CLD21": [
+                "BNBSPOT:PAIR-ADA-USDT"
+                , "BNBSPOT:PAIR-BCH-USDT"
+                , "BNBSPOT:PAIR-BTC-USDT"
+                , "BNBSPOT:PAIR-DOT-USDT"
+                , "BNBSPOT:PAIR-ETH-USDT"
+                , "BNBSPOT:PAIR-LTC-USDT"
+                , "BNBSPOT:PAIR-SOL-USDT"
+                , "BNBSPOT:PAIR-USDC-USDT"
+                , "BNBSPOT:PAIR-XRP-USDT"
+            ]
+            , "BNBSPOT_CVTTDATA": [
+                "BNBSPOT:PAIR-ADA-USDT"
+                , "BNBSPOT:PAIR-BCH-USDT"
+                , "BNBSPOT:PAIR-BTC-USDT"
+                , "BNBSPOT:PAIR-DOT-USDT"
+                , "BNBSPOT:PAIR-ETH-USDT"
+                , "BNBSPOT:PAIR-LTC-USDT"
+                , "BNBSPOT:PAIR-SOL-USDT"
+                , "BNBSPOT:PAIR-USDC-USDT"
+                , "BNBSPOT:PAIR-XRP-USDT"
+            ]
+            , "COINBASE_CLD21": [
+                "COINBASE:PAIR-ADA-USD"
+                , "COINBASE:PAIR-BCH-USD"
+                , "COINBASE:PAIR-BTC-USD"
+                , "COINBASE:PAIR-DOT-USD"
+                , "COINBASE:PAIR-ETH-USD"
+                , "COINBASE:PAIR-LTC-USD"
+                , "COINBASE:PAIR-SOL-USD"
+                , "COINBASE:PAIR-XRP-USD"
+            ]
+            , "COINBASE_CVTTDATA": [
+                "COINBASE:PAIR-ADA-USD"
+                , "COINBASE:PAIR-BCH-USD"
+                , "COINBASE:PAIR-BTC-USD"
+                , "COINBASE:PAIR-DOT-USD"
+                , "COINBASE:PAIR-ETH-USD"
+                , "COINBASE:PAIR-LTC-USD"
+                , "COINBASE:PAIR-SOL-USD"
+                , "COINBASE:PAIR-XRP-USD"
+            ]
+        }
+        , "db": {
+            "class": "cvttpy.tools.db.timescale.tsdb_client.TimescaleDbClient"
+        }
+        , "flush_interval_sec": 0.5
+
+        , "collections": {
+            "__dummy__": null
+            , "ALPACA": {
+                # dynamic instruments names
+                "tables": {
+                    "trades": "alpaca_iex.md_trades"
+                    , "booktops": "alpaca_iex.md_booktops"
+                    , "booksnaps": "alpaca_iex.md_booksnaps"
+                }
+            }
+            , "BNBSPOT": {
+                "instruments": {
+                    "PAIR-ADA-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-BCH-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-BTC-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-DOT-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-ETH-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-LTC-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-SOL-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-USDC-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-XRP-USDT": ["TRADES", "BOOK_TOP"]
+                }
+                , "tables": {
+                    "trades": "bnbspot.md_trades"
+                    , "booktops": "bnbspot.md_booktops"
+                    , "booksnaps": "bnbspot.md_booksnaps"
+                }
+            }
+            , "BNBFUT": {
+                "instruments": {
+                    "PERP-BTC-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PERP-ETH-USDT": ["TRADES", "BOOK_TOP"]
+                }
+                , "tables": {
+                    "trades": "bnbfut.md_trades"
+                    , "booktops": "bnbfut.md_booktops"
+                    , "booksnaps": "bnbfut.md_booksnaps"
+                }
+            }
+            , "COINBASE": {
+                "instruments": {
+                    "PAIR-ADA-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-BCH-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-BTC-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-DOT-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-ETH-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-LTC-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-SOL-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-XRP-USD": ["TRADES", "BOOK_TOP"]
+                }
+                , "tables": {
+                    "trades": "coinbase.md_trades"
+                    , "booktops": "coinbase.md_booktops"
+                    , "booksnaps": "coinbase.md_booksnaps"
+                }
+            }
+        }
+    }
+
+    # ----------------------------------
+    , "exchanges_settings": {
+        "*": {
+            "order_entry": {
+                "enabled": false
+                , "streaming": false
+            },
+            "mkt_data": {
+                "enabled": true
+                , "streaming": true
+            },
+            "accounting": {
+                "enabled": false
+                , "streaming": false
+            }
+        }
+    }
+}
+

market_data/md_recorder/deployment/cvtt-md/.creds

@@ -0,0 +1,17 @@
+{
+    "__dummy__": null
+    , "TSDB_MD_CLD21": {
+        "host": "cloud21.cvtt.vpn"
+        , "port": 5432
+        , "user": "cvtt"
+        , "database": "cvtt_md"
+        , "password": "ICdIh0JnMM7vM7Pf"
+    }
+    , "TSDB_MD_CVTTDATA": {
+        "host": "cvttdata.cvtt.vpn"
+        , "port": 5432
+        , "user": "cvtt"
+        , "database": "cvtt_md"
+        , "password": "ICdIh0JnMM7vM7Pf"
+    }
+}

market_data/md_recorder/deployment/cvtt-md/.env

@@ -0,0 +1,28 @@
+# -------- cvtt-md
+CVTT_USER=1001:1001
+IMAGE_VERSION=0.0.2
+
+CONFIG_FILE=/config/md_recorder.cfg
+CREDS_FILE=/config/.creds
+
+# bnbfut-cvttdata
+BFCD_CONTAINER_NAME=md_recorder.BNBFUT_CVTTDATA
+BFCD_INSTRUMENT_GROUP=BNBFUT_CVTTDATA
+BFCD_EXCHANGE_ACCT_NAME=BNBFUT
+BFCD_DB_CRED_KEY=TSDB_MD_CVTTDATA
+
+BFCD_LOG_FILE=/logs/%T.MD_REC.${BFCD_INSTRUMENT_GROUP}.log
+BFCD_ADD_ARGS=--compress_log
+
+BFCD_ADMIN_PORT=7201
+
+# coinbase-cvttdata
+CBCD_CONTAINER_NAME=md_recorder.COINBASE_CVTTDATA
+CBCD_INSTRUMENT_GROUP=COINBASE_CVTTDATA
+CBCD_EXCHANGE_ACCT_NAME=COINBASE_AT
+CBCD_DB_CRED_KEY=TSDB_MD_CVTTDATA
+
+CBCD_LOG_FILE=/logs/%T.MD_REC.${CBCD_INSTRUMENT_GROUP}.log
+CBCD_ADD_ARGS=--compress_log
+
+CBCD_ADMIN_PORT=7202

market_data/md_recorder/deployment/cvtt-md/docker-compose.yml

@@ -0,0 +1,44 @@
+# CVTT Microservices
+services:
+    bfcd_md_recorder:
+        image: cloud21.cvtt.vpn:5500/md_recorder:${IMAGE_VERSION}
+        container_name: ${BFCD_CONTAINER_NAME}
+        user: ${CVTT_USER:-1001:1001}
+        pull_policy: always
+        restart: unless-stopped
+        environment:
+          CONFIG_SERVICE:     ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          CONFIG_FILE:        ${CONFIG_FILE:-/config/md_recorder.cfg}
+          CREDS_FILE:         ${CREDS_FILE}
+          EXCHANGE_ACCT_NAME: ${BFCD_EXCHANGE_ACCT_NAME}
+          INSTRUMENT_GROUP:   ${BFCD_INSTRUMENT_GROUP}
+          DB_CRED_KEY:        ${BFCD_DB_CRED_KEY}
+          LOG_FILE:           ${BFCD_LOG_FILE}
+          ADD_ARGS:           ${BFCD_ADD_ARGS:-}
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+        ports:
+          - ${BFCD_ADMIN_PORT}:7200
+
+
+    cbcd_md_recorder:
+        image: cloud21.cvtt.vpn:5500/md_recorder:${IMAGE_VERSION}
+        container_name: ${CBCD_CONTAINER_NAME}
+        user: ${CVTT_USER:-1001:1001}
+        pull_policy: always
+        restart: unless-stopped
+        environment:
+          CONFIG_SERVICE:     ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          CONFIG_FILE:        ${CONFIG_FILE:-/config/md_recorder.cfg}
+          CREDS_FILE:         ${CREDS_FILE}
+          EXCHANGE_ACCT_NAME: ${CBCD_EXCHANGE_ACCT_NAME}
+          INSTRUMENT_GROUP:   ${CBCD_INSTRUMENT_GROUP}
+          DB_CRED_KEY:        ${CBCD_DB_CRED_KEY}
+          LOG_FILE:           ${CBCD_LOG_FILE}
+          ADD_ARGS:           ${CBCD_ADD_ARGS:-}
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+        ports:
+          - ${CBCD_ADMIN_PORT}:7200

market_data/md_recorder/deployment/cvtt-md/md_recorder.cfg

@@ -0,0 +1,206 @@
+{
+    "refdata": {
+        "assets": @inc=http://@env{CONFIG_SERVICE}/refdata/assets
+        , "instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/instruments
+        , "exchange_instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/exchange_instruments
+        , "dynamic_instrument_exchanges": ["ALPACA"]
+        , "exchanges": {
+            "COINBASE_AT": {
+                "exchange_id":          "COINBASE"
+                , "name":               "Coinbase Advanced Trade"
+                , "type":               "SPOT"
+
+                , "mktdata" : {
+                    # ****** NEW VAlUE
+                    # "class":  "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMultiWsFeed"
+                    "class":  "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMdFeed"
+                    # ****** NEW VAlUE
+
+                    , "reconnect_secs": 2
+                    , "max_connections": 10
+                    , "price_throttle": true
+
+                    , "ws_url": ["WSS", "wss://ws-feed.exchange.coinbase.com"]
+                }
+            }
+            , "BNBSPOT": {
+                "exchange_id":          "BNBSPOT"
+                , "name":               "Binance Spot"
+                , "type":               "SPOT"
+
+                , "mktdata" : {
+                    # -------- NEW VALUE
+                    # "class":           "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMultiWsFeed"
+                    "class": "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMktDataFeed"
+                    # -------- NEW VALUE
+                    , "max_connections": 10
+                    , "reconnect_secs": 2.0
+
+
+                    , "ws_url": ["WSS", "wss://stream.binance.com/ws"]
+                    , "REST": {
+                        "depth_snapshot_url": ["GET", "https://api.binance.com/api/v3/depth"],
+                    }
+                }
+            }
+            , "BNBFUT": {
+                "exchange_id":          "BNBFUT"
+                , "name":               "Binance Futures USD-M"
+                , "type":               "FUTURES"
+
+                , "mktdata" : {
+                    # -------- NEW VALUE
+                    # "class":           "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMultiWsFeed"
+                    "class":           "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMktDataFeed"
+                    # -------- NEW VALUE
+                    , "max_connections": 10
+                    , "reconnect_secs": 2.0
+
+                    , "ws_url": ["WSS", "wss://fstream.binance.com/ws"]
+
+                    , "REST": {
+                        # https://fapi.binance.com/fapi/v1/depth?symbol=BNBBTC&limit=1000
+                        "depth_snapshot_url": ["GET", "https://fapi.binance.com/fapi/v1/depth"],
+                    }
+                }
+            }
+        }
+    }
+    , "comm": @inc=http://@env{CONFIG_SERVICE}/apps/common/comm
+
+    , "md_recorder": {
+        "instrument_groups": {
+            "BNBFUT_CLD21": [
+                "BNBFUT:PERP-BTC-USDT"
+                , "BNBFUT:PERP-ETH-USDT"
+            ]
+            "BNBFUT_CVTTDATA": [
+                "BNBFUT:PERP-BTC-USDT"
+                , "BNBFUT:PERP-ETH-USDT"
+            ]
+            , "BNBSPOT_CLD21": [
+                "BNBSPOT:PAIR-ADA-USDT"
+                , "BNBSPOT:PAIR-BCH-USDT"
+                , "BNBSPOT:PAIR-BTC-USDT"
+                , "BNBSPOT:PAIR-DOT-USDT"
+                , "BNBSPOT:PAIR-ETH-USDT"
+                , "BNBSPOT:PAIR-LTC-USDT"
+                , "BNBSPOT:PAIR-SOL-USDT"
+                , "BNBSPOT:PAIR-USDC-USDT"
+                , "BNBSPOT:PAIR-XRP-USDT"
+            ]
+            , "BNBSPOT_CVTTDATA": [
+                "BNBSPOT:PAIR-ADA-USDT"
+                , "BNBSPOT:PAIR-BCH-USDT"
+                , "BNBSPOT:PAIR-BTC-USDT"
+                , "BNBSPOT:PAIR-DOT-USDT"
+                , "BNBSPOT:PAIR-ETH-USDT"
+                , "BNBSPOT:PAIR-LTC-USDT"
+                , "BNBSPOT:PAIR-SOL-USDT"
+                , "BNBSPOT:PAIR-USDC-USDT"
+                , "BNBSPOT:PAIR-XRP-USDT"
+            ]
+            , "COINBASE_CLD21": [
+                "COINBASE:PAIR-ADA-USD"
+                , "COINBASE:PAIR-BCH-USD"
+                , "COINBASE:PAIR-BTC-USD"
+                , "COINBASE:PAIR-DOT-USD"
+                , "COINBASE:PAIR-ETH-USD"
+                , "COINBASE:PAIR-LTC-USD"
+                , "COINBASE:PAIR-SOL-USD"
+                , "COINBASE:PAIR-XRP-USD"
+            ]
+            , "COINBASE_CVTTDATA": [
+                "COINBASE:PAIR-ADA-USD"
+                , "COINBASE:PAIR-BCH-USD"
+                , "COINBASE:PAIR-BTC-USD"
+                , "COINBASE:PAIR-DOT-USD"
+                , "COINBASE:PAIR-ETH-USD"
+                , "COINBASE:PAIR-LTC-USD"
+                , "COINBASE:PAIR-SOL-USD"
+                , "COINBASE:PAIR-XRP-USD"
+            ]
+        }
+        , "db": {
+            "class": "cvttpy.tools.db.timescale.tsdb_client.TimescaleDbClient"
+        }
+        , "flush_interval_sec": 0.5
+
+        , "collections": {
+            "__dummy__": null
+            , "ALPACA": {
+                # dynamic instruments names
+                "tables": {
+                    "trades": "alpaca_iex.md_trades"
+                    , "booktops": "alpaca_iex.md_booktops"
+                    , "booksnaps": "alpaca_iex.md_booksnaps"
+                }
+            }
+            , "BNBSPOT": {
+                "instruments": {
+                    "PAIR-ADA-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-BCH-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-BTC-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-DOT-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-ETH-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-LTC-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-SOL-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-USDC-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-XRP-USDT": ["TRADES", "BOOK_TOP"]
+                }
+                , "tables": {
+                    "trades": "bnbspot.md_trades"
+                    , "booktops": "bnbspot.md_booktops"
+                    , "booksnaps": "bnbspot.md_booksnaps"
+                }
+            }
+            , "BNBFUT": {
+                "instruments": {
+                    "PERP-BTC-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PERP-ETH-USDT": ["TRADES", "BOOK_TOP"]
+                }
+                , "tables": {
+                    "trades": "bnbfut.md_trades"
+                    , "booktops": "bnbfut.md_booktops"
+                    , "booksnaps": "bnbfut.md_booksnaps"
+                }
+            }
+            , "COINBASE": {
+                "instruments": {
+                    "PAIR-ADA-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-BCH-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-BTC-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-DOT-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-ETH-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-LTC-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-SOL-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-XRP-USD": ["TRADES", "BOOK_TOP"]
+                }
+                , "tables": {
+                    "trades": "coinbase.md_trades"
+                    , "booktops": "coinbase.md_booktops"
+                    , "booksnaps": "coinbase.md_booksnaps"
+                }
+            }
+        }
+    }
+
+    # ----------------------------------
+    , "exchanges_settings": {
+        "*": {
+            "order_entry": {
+                "enabled": false
+                , "streaming": false
+            },
+            "mkt_data": {
+                "enabled": true
+                , "streaming": true
+            },
+            "accounting": {
+                "enabled": false
+                , "streaming": false
+            }
+        }
+    }
+}
+

market_data/md_recorder/entrypoint.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# runs in container
+
+
+Cmd="python3.12"
+Cmd+=" cvttpy/apps/md/md_recorder.py"
+
+Cmd+=" --config=${CONFIG_FILE:-/config/cvtt_musvc.cfg}"
+Cmd+=" --credentials_file=${CREDS_FILE:-/config/.creds}"
+
+Cmd="${Cmd} --instrument_group=${INSTRUMENT_GROUP}"
+Cmd="${Cmd} --exch_acct_cfgname=${EXCHANGE_ACCT_NAME}"
+Cmd="${Cmd} --db_credentials_key=${DB_CRED_KEY}"
+Cmd="${Cmd} --admin_port=7200"
+
+Cmd+=" --log_file=${LOG_FILE:-/logs/%T.MD_REC.${INSTRUMENT_GROUP}.log}"
+Cmd+=" ${ADD_ARGS}"
+
+echo ${Cmd}
+exec ${Cmd}
+

market_data/md_recorder/requirements.txt

@@ -5,3 +5,4 @@ hjson>=3.1.0
 pandas>=1.5.3
 sortedcontainers>=2.4.0
 redis>=5.0.8
+debugpy>=1.8.13

market_data/md_recorder/run_md_recorder.sh

@@ -1,69 +0,0 @@
-#!/bin/bash
-
-# runs in container
-
-ValidJobs=('BNBFUT_CLD21' 'BNBSPOT_CLD21' 'COINBASE_CLD21' 'BNBFUT_CVTTDATA' 'BNBSPOT_CVTTDATA' 'COINBASE_CVTTDATA')
-# runs on host to start container
-usage() {
-  echo "Usage: $0 <job_name; one of (${ValidJobs[@]})>"
-  exit 1
-}
-
-is_valid() {
-  local job=$1
-  for valid_job in "${ValidJobs[@]}";
-  do
-    # echo "job=$job valid_job=$valid_job"
-    if [[ "${job}" == "${valid_job}" ]]; then
-      return 0
-    fi
-  done
-  return 1
-}
-
-job=${1}
-if  ! is_valid "${job}"; then
-    usage
-fi
-
-ConfigServer=cloud23.cvtt.vpn:6789
-
-Cmd="python3.12"
-Cmd="${Cmd} cvttpy/apps/md/md_recorder.py"
-Cmd="${Cmd} --config=http://${ConfigServer}/apps/md_recorder"
-Cmd="${Cmd} --credentials_file=/.creds"
-Cmd="${Cmd} --compress_log"
-Cmd="${Cmd} --instrument_group=${job}"
-Cmd="${Cmd} --log_file=/logs/%T.MD_REC.${job}.log"
-if [ "${job}" == "BNBFUT_CLD21" ] ; then
-    Cmd="${Cmd} --active_exchanges=BNBFUT"
-    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
-    Cmd="${Cmd} --admin_port=7201"
-elif [ "${job}" == "COINBASE_CLD21" ] ; then
-    Cmd="${Cmd} --active_exchanges=COINBASE_AT"
-    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
-    Cmd="${Cmd} --admin_port=7202"
-elif [ "${job}" == "BNBSPOT_CLD21" ] ; then
-    Cmd="${Cmd} --active_exchanges=BNBSPOT"
-    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
-    Cmd="${Cmd} --admin_port=7203"
-elif [ "${job}" == "BNBSPOT_CVTTDATA" ] ; then
-    Cmd="${Cmd} --active_exchanges=BNBSPOT"
-    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
-    Cmd="${Cmd} --admin_port=7204"
-elif [ "${job}" == "BNBFUT_CVTTDATA" ] ; then
-    Cmd="${Cmd} --active_exchanges=BNBFUT"
-    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
-    Cmd="${Cmd} --admin_port=7205"
-elif [ "${job}" == "COINBASE_CVTTDATA" ] ; then
-    Cmd="${Cmd} --active_exchanges=COINBASE_AT"
-    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
-    Cmd="${Cmd} --admin_port=7206"
-else
-    echo "Unrecognized JOB: ${job}"
-    exit 1
-fi
-echo ${Cmd}
-eval ${Cmd}
-
-

market_data/md_recorder_monitor/run_mdr_monitor.sh

@@ -32,7 +32,7 @@ if [ "${2}" != "" ]; then
 fi
 
 
-ConfigServer=cloud23.cvtt.vpn:6789
+ConfigServer=cloud16.cvtt.vpn:6789
 
 Cmd="python3.12"
 Cmd="${Cmd} cvttpy/apps/md/md_rec_monitor.py"

market_data/relative_liquidity/.creds

@@ -0,0 +1,10 @@
+{
+    "__dummy": null
+    , "INFISICAL": {
+        "url": "https://infisical.cvtt.net"
+        , "client_id": "e330a1c4-08dc-452f-ac02-5b37fff4ab3e"
+        , "client_secret": "190554828c3c342f63517320ac029f822b6358aa6187aec266465d0bde4d6b3c"
+        , "workspace_id": "6923c950-8473-4cec-818f-10df41d8b39c"
+        , "environment": "dev"
+    }
+}

market_data/relative_liquidity/Dockerfile

@@ -1,7 +1,10 @@
 FROM python:3.12-slim
+ARG FROM_DIR=docker_dev/market_data/relative_liquidity
 
 WORKDIR /
-COPY docker_dev/relative_liquidity/requirements.txt /
+COPY ${FROM_DIR}/.creds  /.creds
+COPY ${FROM_DIR}/requirements.txt /requirements.txt
+
 RUN pip install --upgrade pip --root-user-action=ignore
 RUN pip install -r /requirements.txt --root-user-action=ignore
 
@@ -15,19 +18,9 @@ ENV PYTHONPATH=/
 CMD [                                                            \
     "python3.12",                                                \
     "cvttpy/apps/md/rel_lqdt_server.py",                         \
+    "--config=http://cloud16.cvtt.vpn:6789/apps/minimal_md_eqt", \
+    "--credentials_file=/.creds",                                \
     "--port=5678",                                               \
     "--db_file=/app/data/rel_liquidity.db",                      \
-    "--log_file=/logs/%T.REL_LIQUIDITY_SVC.log" \
+    "--log_file=/logs/%D.REL_LIQUIDITY_SVC.log"                  \
 ]
-
-
-# Cmd="docker run"
-# Cmd="${Cmd} -d"
-# Cmd="${Cmd} --rm"
-# Cmd="${Cmd} --pull=always"
-# Cmd="${Cmd} --name=relative_liquidity_svc"
-# Cmd="${Cmd} -p 5678:5678"
-# Cmd="${Cmd} -v /home/cvtt/prod/data:/app/data"
-# Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
-# Cmd="${Cmd} cloud21.cvtt.vpn:5500/relative_liquidity:latest"
-

market_data/relative_liquidity/requirements.txt

@@ -3,4 +3,4 @@ nest-asyncio>=1.5.5
 numpy>=1.24.1
 hjson>=3.1.0
 pandas>=1.5.3
-# sortedcontainers>=2.4.0
+sortedcontainers>=2.4.0

microservices/.env

@@ -0,0 +1,38 @@
+CONFIG_SERVICE=cloud16.cvtt.vpn:6789
+CONFIG_COMM_SETTINGS=comm/pubsub/redis/docker_microservices
+CONFIG_FILE="http://${CONFIG_SERVICE}/apps/cvtt_musvc"
+
+CVTT_USER=1001:1001
+CVTT_VERSION=0.0.3
+
+# MDGW_CREDS_FILE=.creds
+MDGW_LOG_FILE=%T.md_gateway.log
+MDGW_ADD_ARGS=
+
+# OEGW_CREDS_FILE=.creds
+OEGW_LOG_FILE=%T.oe_gateway.log
+#OEGW_ADD_ARGS="--log_level=DEBUG --debug_mode"
+OEGW_ADD_ARGS="--debug_mode"
+
+# ACGW_CREDS_FILE=.creds
+ACGW_LOG_FILE=%T.ac_gateway.log
+ACGW_ADD_ARGS=
+
+PXER_LOG_FILE=%T.pricer.log
+PXER_ADD_ARGS=
+
+COORD_LOG_FILE=%T.coordinator.log
+COORD_ADD_ARGS=
+
+BKPR_CREDS_FILE=
+BKPR_LOG_FILE=%T.bookkeeper.log
+BKPR_ADD_ARGS=
+
+RMGR_LOG_FILE=%T.riskmrg.log
+RMGR_ADD_ARGS=
+
+QUANT_LOG_FILE=%T.quant.log
+QUANT_ADD_ARGS=
+
+TRDR_LOG_FILE=%T.trader.log
+TRDR_ADD_ARGS=

microservices/ac_gateway/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.12-slim
+
+COPY requirements.txt /
+
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+
+RUN mkdir -p /logs /config
+
+SHELL ["/bin/bash", "-c"]
+
+ENV PYTHONPATH=/
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

microservices/ac_gateway/entrypoint.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+LogDir=/logs
+
+Cmd="python3.12"
+Cmd+=" cvttpy/apps/microservices/market_gateways/accounting_gateway.py"
+
+Cmd+=" --config=${CONFIG_FILE:-/config/cvtt_musvc.cfg}"
+Cmd+=" --credentials_file=${ACGW_CREDS_FILE:-/config/.creds}"
+
+Cmd+=" --log_file=${LogDir}/${ACGW_LOG_FILE:-%T.ac_gateway.log}"
+Cmd+=" ${ACGW_ADD_ARGS}"
+
+echo ${Cmd}
+exec ${Cmd}
+

microservices/ac_gateway/requirements.txt

@@ -5,3 +5,5 @@ sortedcontainers>=2.4.0
 redis>=5.0.8
 python-dateutil>=2.8.2
 types-python-dateutil>=2.8.19.6
+cryptography>=43.0.0
+PyJWT>=2.10.1

microservices/bookkeeper/Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.12-slim
+
+# libpq needed for Timescale
+RUN apt-get update && apt-get install -y \
+    apt-utils \
+    libpq-dev \
+    build-essential
+
+    COPY requirements.txt /
+
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+
+RUN mkdir -p /logs /config
+
+SHELL ["/bin/bash", "-c"]
+
+ENV PYTHONPATH=/
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

microservices/bookkeeper/entrypoint.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+LogDir=/logs
+
+Cmd="python3.12"
+Cmd+=" cvttpy/apps/microservices/book_keeper/book_keeper.py"
+
+Cmd+=" --config=${CONFIG_FILE}"
+Cmd+=" --log_file=${LogDir}/${BKPR_LOG_FILE}"
+Cmd+=" --credentials_file=${BKPR_CREDS_FILE:-/config/.creds}"
+
+
+Cmd+=" ${BKPR_ADD_ARGS}"
+echo ${Cmd}
+exec ${Cmd}
+

microservices/bookkeeper/requirements.txt

@@ -2,14 +2,10 @@ aiohttp>=3.7.4.post0
 nest-asyncio>=1.5.5
 hjson>=3.1.0
 sortedcontainers>=2.4.0
-pandas>=1.5.3
-#
 redis>=5.0.8
-#
 python-dateutil>=2.8.2
 types-python-dateutil>=2.8.19.6
-#
+cryptography>=43.0.0
+PyJWT>=2.10.1
 psycopg>=3.2.1
-psycopg-binary>=3.2.1
-psycopg2-binary>=2.9.6
-
+pandas>=1.5.3

microservices/build.sh

@@ -0,0 +1,106 @@
+#!/bin/bash
+
+# --- Settings
+RegistryService=cloud21.cvtt.vpn:5500
+ValidImages=(md_gateway)
+ValidImages=(oe_gateway)
+ValidImages=(ac_gateway)
+ValidImages=(pricer)
+ValidImages=(coordinator)
+ValidImages=(bookkeeper)
+ValidImages+=(risk_manager)
+ValidImages+=(quant)
+ValidImages+=(trader)
+BashFunctionCode=$(realpath $(dirname ${0}))/../dckr_bld_functions.sh
+# --- Settings
+
+usage() {
+  echo "Usage: $0 -I <image_name> -V <image_version> -L <library>:<library_version>"
+  exit 1
+}
+
+
+# ---------------- cmdline
+while getopts "I:V:L:" opt; do
+  case ${opt} in
+    I )
+      ImageName=$OPTARG
+      ;;
+    V )
+      ImageVersion=$OPTARG
+      ;;
+    L )
+      Library=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+# ---------------- cmdline
+
+if [ "${ImageName}" == "" ]
+then
+    usage
+fi
+
+if [ "${ImageVersion}" == "" ]
+then
+    usage
+fi
+
+Project=
+ProjectVersion=
+if  [ "${Library}" != "" ] ; then
+    IFS=':' read -ra parts <<< "${Library}"
+    Project=${parts[0]}
+    ProjectVersion=${parts[1]}
+fi
+# exit
+
+build_musvc_image() {
+    ImageName=${1}
+    Project=${2}
+    ProjectVersion=${3}
+
+    DockerDir=$(realpath $(dirname ${0})/${ImageName})
+    cd ${DockerDir}
+
+    if [ "${Project}" != "" ]; then
+        get_project_version ${Project} ${ProjectVersion}
+    fi
+
+    function cleanup {
+        cd ${DockerDir}
+        rm -rf cvttpy
+    }
+    trap cleanup RETURN
+
+    build_docker_image ${ImageName} ${RegistryService} ${ImageVersion} ${Project} ${ProjectVersion}
+}
+
+
+source ${BashFunctionCode}
+if [ "${ImageName}" == "__ALL__" ]; then
+    Images=(${ValidImages[@]})
+else
+    if [[ " ${ValidImages[*]} " == *" ${ImageName}"* ]]; then
+        Images=(${ImageName})
+    else
+        echo "ERROR: invalid image name \"${ImageName}\""
+        exit 1
+    fi
+fi
+for ImageName in "${Images[@]}"; do
+    build_musvc_image ${ImageName} ${Project} ${ProjectVersion}
+done
+
+# echo "**** D E B U G  E X I T" && exit
+echo "***** ${0}  D O N E"
+
+

microservices/coordinator/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.12-slim
+
+COPY requirements.txt /
+
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+
+RUN mkdir -p /logs /config
+
+SHELL ["/bin/bash", "-c"]
+
+ENV PYTHONPATH=/
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

microservices/coordinator/entrypoint.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+LogDir=/logs
+
+Cmd="python3.12"
+Cmd+=" cvttpy/apps/microservices/coordinator/coordinator.py"
+
+Cmd+=" --config=${CONFIG_FILE:-/config/cvtt_musvc.cfg}"
+
+Cmd+=" --log_file=${LogDir}/${COORD_LOG_FILE:-%T.coordinator.log}"
+Cmd+=" ${COORD_ADD_ARGS}"
+echo ${Cmd}
+exec ${Cmd}
+

microservices/coordinator/requirements.txt

@@ -5,3 +5,5 @@ sortedcontainers>=2.4.0
 redis>=5.0.8
 python-dateutil>=2.8.2
 types-python-dateutil>=2.8.19.6
+cryptography>=43.0.0
+PyJWT>=2.10.1

microservices/docker-compose.yml

@@ -1,52 +1,183 @@
-version: "3.9"
-
+# CVTT Microservices
 services:
-    md_gateway:
-        image: cloud21.cvtt.vpn:5500/md_gateway:latest
-        container_name: md_gateway
+    trader:
+        image: cloud21.cvtt.vpn:5500/trader:${CVTT_VERSION}
+        container_name: cvtt-trader
+        pull_policy: always
+        user: ${CVTT_USER:-1001:1001}
         environment:
-          - REDIS_HOST=redis
+          CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          CONFIG_COMM_SETTINGS: ${CONFIG_COMM_SETTINGS:-comm/pubsub/redis/docker_microservices}
+          CONFIG_FILE:  ${CONFIG_FILE:-http://${CONFIG_SERVICE}/apps/cvtt_musvc}
+          TRDR_LOG_FILE:  ${TRDR_LOG_FILE:-%T.trader.log}
+          TRDR_ADD_ARGS: ${TRDR_ADD_ARGS:-}
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+        depends_on:
+          - md_gateway
+          - oe_gateway
+          - quant
+          - bookkeeper
+          - risk_manager
+          - redis
+
+    quant:
+        image: cloud21.cvtt.vpn:5500/quant:${CVTT_VERSION}
+        container_name: cvtt-quant
+        pull_policy: always
+        user: ${CVTT_USER:-1001:1001}
+        environment:
+          CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          CONFIG_COMM_SETTINGS: ${CONFIG_COMM_SETTINGS:-comm/pubsub/redis/docker_microservices}
+          CONFIG_FILE:  ${CONFIG_FILE:-http://${CONFIG_SERVICE}/apps/cvtt_musvc}
+          QUANT_LOG_FILE:  ${QUANT_LOG_FILE:-%T.quant.log}
+          QUANT_ADD_ARGS: ${QUANT_ADD_ARGS:-}
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+        depends_on:
+          - md_gateway
+          - bookkeeper
+          - redis
+
+    coordinator:
+        image: cloud21.cvtt.vpn:5500/coordinator:${CVTT_VERSION}
+        container_name: cvtt-coordinator
+        pull_policy: always
+        user: ${CVTT_USER:-1001:1001}
+        environment:
+          CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          CONFIG_COMM_SETTINGS: ${CONFIG_COMM_SETTINGS:-comm/pubsub/redis/docker_microservices}
+          CONFIG_FILE:  ${CONFIG_FILE:-cvtt_musvc.cfg}
+          COORD_LOG_FILE:  ${COORD_LOG_FILE:-%T.coordinator.log}
+          COORD_ADD_ARGS: ${COORD_ADD_ARGS:-}
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+        depends_on:
+          - pricer
+          # - quant
+          # - trader
+          - risk_manager
+          - redis
+
+    pricer:
+        image: cloud21.cvtt.vpn:5500/pricer:${CVTT_VERSION}
+        container_name: cvtt-pricer
+        pull_policy: always
+        user: ${CVTT_USER:-1001:1001}
+        environment:
+          CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          CONFIG_COMM_SETTINGS: ${CONFIG_COMM_SETTINGS:-comm/pubsub/redis/docker_microservices}
+          CONFIG_FILE:  ${CONFIG_FILE:-http://${CONFIG_SERVICE}/apps/cvtt_musvc}
+          PXER_LOG_FILE:  ${PXER_LOG_FILE:-%T.pricer.log}
+          PXER_ADD_ARGS: ${PXER_ADD_ARGS:-}
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+        depends_on:
+          - md_gateway
+          - redis
+
+    risk_manager:
+        image: cloud21.cvtt.vpn:5500/risk_manager:${CVTT_VERSION}
+        container_name: cvtt-risk_manager
+        pull_policy: always
+        user: ${CVTT_USER:-1001:1001}
+        environment:
+          CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          CONFIG_COMM_SETTINGS: ${CONFIG_COMM_SETTINGS:-comm/pubsub/redis/docker_microservices}
+          CONFIG_FILE:  ${CONFIG_FILE:-http://${CONFIG_SERVICE}/apps/cvtt_musvc}
+          RMGR_LOG_FILE:  ${RMGR_LOG_FILE:-%T.riskmrg.log}
+          RMGR_ADD_ARGS: ${RMGR_ADD_ARGS:-}
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+        depends_on:
+          - md_gateway
+          - ac_gateway
+          - bookkeeper
+          - redis
+
+
+    bookkeeper:
+        image: cloud21.cvtt.vpn:5500/bookkeeper:${CVTT_VERSION}
+        container_name: cvtt-bookkeeper
+        pull_policy: always
+        user: ${CVTT_USER:-1001:1001}
+        environment:
+          CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          CONFIG_COMM_SETTINGS: ${CONFIG_COMM_SETTINGS:-comm/pubsub/redis/docker_microservices}
+          CONFIG_FILE:  ${CONFIG_FILE:-http://${CONFIG_SERVICE}/apps/cvtt_musvc}
+          BKPR_LOG_FILE:  ${BKPR_LOG_FILE:-%T.bookkeeper.log}
+          BKPR_ADD_ARGS: ${BKPR_ADD_ARGS:-}
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+        depends_on:
+          - ac_gateway
+          #- trader
+          - redis
+
+    md_gateway:
+        image: cloud21.cvtt.vpn:5500/md_gateway:${CVTT_VERSION}
+        container_name: md_gateway
+        pull_policy: always
+        user: ${CVTT_USER:-1001:1001}
+        environment:
+          CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          CONFIG_COMM_SETTINGS: ${CONFIG_COMM_SETTINGS:-comm/pubsub/redis/docker_microservices}
+          CONFIG_FILE:  ${CONFIG_FILE:-http://${CONFIG_SERVICE}/apps/cvtt_musvc}
+          MDGW_CREDS_FILE:  ${MDGW_CREDS_FILE:-/config/.creds}
+          MDGW_LOG_FILE:  ${MDGW_LOG_FILE:-%T.md_gateway.log}
+          MDGW_ADD_ARGS: ${MDGW_ADD_ARGS:-}
         volumes:
-          - ./.creds:/.creds
           - ./config:/config
           - ./logs:/logs
-          - ./data:/data
         depends_on:
           - redis
 
-    tester:
-        image: cloud21.cvtt.vpn:5500/tester:latest
-        container_name: tester
+    oe_gateway:
+        image: cloud21.cvtt.vpn:5500/oe_gateway:${CVTT_VERSION}
+        container_name: oe_gateway
+        pull_policy: always
+        user: ${CVTT_USER:-1001:1001}
         environment:
-          - REDIS_HOST=redis
+          CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          CONFIG_COMM_SETTINGS: ${CONFIG_COMM_SETTINGS:-comm/pubsub/redis/docker_microservices}
+          CONFIG_FILE:  ${CONFIG_FILE:-http://${CONFIG_SERVICE}/apps/cvtt_musvc}
+          OEGW_CREDS_FILE:  ${OEGW_CREDS_FILE:-/config/.creds}
+          OEGW_LOG_FILE:  ${OEGW_LOG_FILE:-%T.oe_gateway.log}
+          OEGW_ADD_ARGS: ${OEGW_ADD_ARGS:-}
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+        depends_on:
+          - redis
+
+    ac_gateway:
+        image: cloud21.cvtt.vpn:5500/ac_gateway:${CVTT_VERSION}
+        container_name: ac_gateway
+        pull_policy: always
+        user: ${CVTT_USER:-1001:1001}
+        environment:
+          CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          CONFIG_COMM_SETTINGS: ${CONFIG_COMM_SETTINGS:-comm/pubsub/redis/docker_microservices}
+          CONFIG_FILE:  ${CONFIG_FILE:-http://${CONFIG_SERVICE}/apps/cvtt_musvc}
+          ACGW_CREDS_FILE:  ${ACGW_CREDS_FILE:-/config/.creds}
+          ACGW_LOG_FILE:  ${ACGW_LOG_FILE:-%T.ac_gateway.log}
+          ACGW_ADD_ARGS: ${ACGW_ADD_ARGS:-}
         volumes:
           - ./config:/config
           - ./logs:/logs
-          - ./data:/data
         depends_on:
           - redis
-          - md_gateway
 
     redis:
         image: redis:latest
-        container_name: redis
+        container_name: redis-cvtt
         ports:
-          - "6379:6379"
+          - "16379:6379"
         volumes:
           - ./data/redis:/data
-
-
-#   test_program:
-#     image: your_test_program_image:latest
-#     container_name: test_program
-#     environment:
-#       - REDIS_HOST=redis
-#     volumes:
-#       - ./config:/shared/config
-#       - ./logs:/shared/logs
-#     depends_on:
-#       - market_data_gateway
-#       - redis
-
-# volumes:
-#   redis_data:

microservices/md_gateway/Dockerfile

@@ -1,25 +1,19 @@
 FROM python:3.12-slim
 
-ARG ROOT_MCRSVC_DIR=docker_dev/microservices
-ARG FROM_DIR=${ROOT_MCRSVC_DIR}/md_gateway
+COPY requirements.txt /
 
-COPY ${ROOT_MCRSVC_DIR}/requirements.txt /
 RUN pip install --upgrade pip --root-user-action=ignore
 RUN pip install -r /requirements.txt --root-user-action=ignore
 
 COPY cvttpy /cvttpy
-# COPY ${ROOT_MCRSVC_DIR}/.creds  /.creds
 
+RUN mkdir -p /logs /config
 
-COPY ${FROM_DIR}/run_mdgw.sh /run_mdgw.sh
-RUN chmod +x /run_mdgw.sh
-
-RUN mkdir /logs
-RUN mkdir -p /config
-
-
-WORKDIR /
 SHELL ["/bin/bash", "-c"]
-ENV PYTHONPATH=/
-ENTRYPOINT [ "/run_mdgw.sh" ]
 
+ENV PYTHONPATH=/
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

microservices/md_gateway/entrypoint.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+LogDir=/logs
+
+Cmd="python3.12"
+Cmd+=" cvttpy/apps/microservices/market_gateways/md_gateway.py"
+
+Cmd+=" --config=${CONFIG_FILE:-/config/cvtt_musvc.cfg}"
+Cmd+=" --credentials_file=${MDGW_CREDS_FILE:-/config/.creds}"
+
+Cmd+=" --log_file=${LogDir}/${MDGW_LOG_FILE:-%T.md_gateway.log}"
+Cmd+=" ${MDGW_ADD_ARGS}"
+echo ${Cmd}
+exec ${Cmd}
+

microservices/md_gateway/requirements.txt

@@ -5,3 +5,5 @@ sortedcontainers>=2.4.0
 redis>=5.0.8
 python-dateutil>=2.8.2
 types-python-dateutil>=2.8.19.6
+cryptography>=43.0.0
+PyJWT>=2.10.1

microservices/md_gateway/run_mdgw.sh

@@ -1,48 +0,0 @@
-#!/bin/bash
-
-# runs in container
-
-# runs on host to start container
-usage() {
-    echo -n "Usage: ${0}"
-    echo -n " [-c <config (/config/cvtt.cfg)>]"
-    echo -n " [-a <admin_port (7220)>]"
-    echo -n " [-z (compress log)"]
-    echo
-    exit 1
-}
-
-Config=/config/cvtt.cfg
-AdminPort=7220
-COMPRESS_LOG=false
-
-
-while getopts "c:a:z" opt; do
-    case ${opt} in
-        c ) Config=$OPTARG  ;;
-        a ) AdminPort=$OPTARG ;;
-        z ) COMPRESS_LOG=true ;;
-        \? )
-            echo "Invalid option: -$OPTARG" >&2
-            usage
-            ;;
-        : )
-            echo "Option -$OPTARG requires an argument." >&2
-            usage
-            ;;
-    esac
-done
-
-Cmd="python3.12"
-Cmd="${Cmd} cvttpy/apps/microservices/market_gateways/md_gateway.py"
-Cmd="${Cmd} --config=${Config}"
-Cmd="${Cmd} --credentials_file=/.creds"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_file=/logs/%T.md_gateway.log"
-if ${COMPRESS_LOG} ; then
-    Cmd="${Cmd} --compress_log"
-fi
-echo ${Cmd}
-eval ${Cmd}
-
-

microservices/oe_gateway/Dockerfile

@@ -1,25 +1,19 @@
 FROM python:3.12-slim
 
-ARG ROOT_MCRSVC_DIR=docker_dev/microservices
-ARG FROM_DIR=${ROOT_MCRSVC_DIR}/oe_gateway
+COPY requirements.txt /
 
-COPY ${ROOT_MCRSVC_DIR}/requirements.txt /
 RUN pip install --upgrade pip --root-user-action=ignore
 RUN pip install -r /requirements.txt --root-user-action=ignore
 
 COPY cvttpy /cvttpy
-COPY ${ROOT_MCRSVC_DIR}/.creds  /.creds
 
+RUN mkdir -p /logs /config
 
-COPY ${FROM_DIR}/run_oegw.sh /run_oegw.sh
-RUN chmod +x /run_oegw.sh
-
-RUN mkdir /logs
-RUN mkdir -p /config
-
-
-WORKDIR /
 SHELL ["/bin/bash", "-c"]
-ENV PYTHONPATH=/
-ENTRYPOINT [ "/run_oegw.sh" ]
 
+ENV PYTHONPATH=/
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

microservices/oe_gateway/entrypoint.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+LogDir=/logs
+
+Cmd="python3.12"
+Cmd+=" cvttpy/apps/microservices/market_gateways/order_entry_gateway.py"
+
+Cmd+=" --config=${CONFIG_FILE:-/config/cvtt_musvc.cfg}"
+Cmd+=" --credentials_file=${OEGW_CREDS_FILE:-/config/.creds}"
+
+Cmd+=" --log_file=${LogDir}/${OEGW_LOG_FILE:-%T.oe_gateway.log}"
+Cmd+=" ${OEGW_ADD_ARGS}"
+echo ${Cmd}
+exec ${Cmd}
+

microservices/oe_gateway/requirements.txt

@@ -2,14 +2,10 @@ aiohttp>=3.7.4.post0
 nest-asyncio>=1.5.5
 hjson>=3.1.0
 sortedcontainers>=2.4.0
-pandas>=1.5.3
-#
 redis>=5.0.8
-#
 python-dateutil>=2.8.2
 types-python-dateutil>=2.8.19.6
-#
-psycopg>=3.2.1
-psycopg-binary>=3.2.1
-psycopg2-binary>=2.9.6
+cryptography>=43.0.0
+PyJWT>=2.10.1
+
 

microservices/oe_gateway/run_oegw.sh

@@ -1,48 +0,0 @@
-#!/bin/bash
-
-# runs in container
-
-# runs on host to start container
-usage() {
-    echo -n "Usage: ${0}"
-    echo -n " [-c <config (/config/cvtt.cfg)>]"
-    echo -n " [-a <admin_port (7220)>]"
-    echo -n " [-z (compress log)"]
-    echo
-    exit 1
-}
-
-Config=/config/cvtt.cfg
-AdminPort=7220
-COMPRESS_LOG=false
-
-
-while getopts "c:a:z" opt; do
-    case ${opt} in
-        c ) Config=$OPTARG  ;;
-        a ) AdminPort=$OPTARG ;;
-        z ) COMPRESS_LOG=true ;;
-        \? )
-            echo "Invalid option: -$OPTARG" >&2
-            usage
-            ;;
-        : )
-            echo "Option -$OPTARG requires an argument." >&2
-            usage
-            ;;
-    esac
-done
-
-Cmd="python3.12"
-Cmd="${Cmd} cvttpy/apps/microservices/market_gateways/order_entry_gateway.py"
-Cmd="${Cmd} --config=${Config}"
-Cmd="${Cmd} --credentials_file=/.creds"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_file=/logs/%T.md_gateway.log"
-if ${COMPRESS_LOG} ; then
-    Cmd="${Cmd} --compress_log"
-fi
-echo ${Cmd}
-eval ${Cmd}
-
-

microservices/pricer/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.12-slim
+
+COPY requirements.txt /
+
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+
+RUN mkdir -p /logs /config
+
+SHELL ["/bin/bash", "-c"]
+
+ENV PYTHONPATH=/
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

microservices/pricer/entrypoint.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+LogDir=/logs
+
+Cmd="python3.12"
+Cmd+=" cvttpy/apps/microservices/pricer/pricer.py"
+
+Cmd+=" --config=${CONFIG_FILE:-/config/cvtt_musvc.cfg}"
+Cmd+=" --log_file=${LogDir}/${PXER_LOG_FILE:-%T.pricer.log}"
+
+Cmd+=" ${PXER_ADD_ARGS}"
+echo ${Cmd}
+exec ${Cmd}
+

microservices/pricer/requirements.txt

@@ -0,0 +1,9 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5
+hjson>=3.1.0
+sortedcontainers>=2.4.0
+redis>=5.0.8
+python-dateutil>=2.8.2
+types-python-dateutil>=2.8.19.6
+cryptography>=43.0.0
+PyJWT>=2.10.1

microservices/quant/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.12-slim
+
+COPY requirements.txt /
+
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+
+RUN mkdir -p /logs /config
+
+SHELL ["/bin/bash", "-c"]
+
+ENV PYTHONPATH=/
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

microservices/quant/entrypoint.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+LogDir=/logs
+
+Cmd="python3.12"
+Cmd+=" cvttpy/apps/microservices/quant/quant.py"
+
+Cmd+=" --config=${CONFIG_FILE:-/config/cvtt_musvc.cfg}"
+Cmd+=" --log_file=${LogDir}/${QUANT_LOG_FILE}"
+
+Cmd+=" ${QUANT_ADD_ARGS}"
+echo ${Cmd}
+exec ${Cmd}
+

microservices/quant/requirements.txt

@@ -0,0 +1,9 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5
+hjson>=3.1.0
+sortedcontainers>=2.4.0
+redis>=5.0.8
+python-dateutil>=2.8.2
+types-python-dateutil>=2.8.19.6
+cryptography>=43.0.0
+PyJWT>=2.10.1

microservices/risk_manager/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.12-slim
+
+COPY requirements.txt /
+
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+
+RUN mkdir -p /logs /config
+
+SHELL ["/bin/bash", "-c"]
+
+ENV PYTHONPATH=/
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

microservices/risk_manager/entrypoint.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+LogDir=/logs
+
+Cmd="python3.12"
+Cmd+=" cvttpy/apps/microservices/risk_manager/risk_manager.py"
+
+Cmd+=" --config=${CONFIG_FILE}"
+Cmd+=" --log_file=${LogDir}/${RMGR_LOG_FILE}"
+
+Cmd+=" ${RMGR_ADD_ARGS}"
+echo ${Cmd}
+exec ${Cmd}
+

microservices/risk_manager/requirements.txt

@@ -0,0 +1,9 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5
+hjson>=3.1.0
+sortedcontainers>=2.4.0
+redis>=5.0.8
+python-dateutil>=2.8.2
+types-python-dateutil>=2.8.19.6
+cryptography>=43.0.0
+PyJWT>=2.10.1

microservices/tester/Dockerfile

@@ -1,26 +0,0 @@
-FROM python:3.12-slim
-
-ARG ROOT_MUSVC_DIR=docker_dev/microservices
-ARG FROM_DIR=${ROOT_MUSVC_DIR}/tester
-
-COPY ${ROOT_MUSVC_DIR}/requirements.txt /
-RUN pip install --upgrade pip --root-user-action=ignore
-RUN pip install -r /requirements.txt --root-user-action=ignore
-
-COPY cvttpy /cvttpy
-# COPY ${ROOT_MUSVC_DIR}/.creds  /.creds
-
-
-COPY ${FROM_DIR}/run_tester.sh /run_tester.sh
-RUN chmod +x /run_tester.sh
-
-RUN mkdir /logs
-RUN mkdir -p /config
-RUN mkdir -p /data
-
-
-WORKDIR /
-SHELL ["/bin/bash", "-c"]
-ENV PYTHONPATH=/
-ENTRYPOINT [ "/run_tester.sh" ]
-

microservices/tester/run_tester.sh

@@ -1,55 +0,0 @@
-#!/bin/bash
-
-# runs in container
-
-# runs on host to start container
-usage() {
-    echo -n "Usage: ${0}"
-    echo -n " [-c <config (/config/cvtt.cfg) >]"
-    echo -n " [-i <instruments (COINBASE:PAIR-BTC-USD,BNBFUT:PERP-BTC-USDT) >]"
-    echo -n " [-d <data types (TRADES) >]"
-    echo -n " [-a <admin_port (7221) >]"
-    echo -n " [-z (compress log (false))"]
-    echo
-    exit 1
-}
-
-Config=/config/cvtt.cfg
-AdminPort=7221
-COMPRESS_LOG=false
-Instruments="COINBASE:PAIR-BTC-USD,BNBFUT:PERP-BTC-USDT"
-DataTypes="TRADES"
-
-while getopts "a:c:d:i:z" opt; do
-    case ${opt} in
-        c ) Config=$OPTARG  ;;
-        a ) AdminPort=$OPTARG ;;
-        i ) Instruments=$OPTARG ;;
-        d ) DataTypes=$OPTARG ;;
-        z ) COMPRESS_LOG=true ;;
-        \? )
-            echo "Invalid option: -$OPTARG" >&2
-            usage
-            ;;
-        : )
-            echo "Option -$OPTARG requires an argument." >&2
-            usage
-            ;;
-    esac
-done
-
-Cmd="python3.12"
-Cmd="${Cmd} cvttpy/apps/microservices/tester/musvc_tester.py"
-Cmd="${Cmd} --config=${Config}"
-Cmd="${Cmd} --credentials_file=/.creds"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --data_types=${DataTypes}"
-Cmd="${Cmd} --instruments=${Instruments}"
-Cmd="${Cmd} --log_file=/logs/%T.tester.log"
-if ${COMPRESS_LOG} ; then
-    Cmd="${Cmd} --compress_log"
-fi
-echo ${Cmd}
-eval ${Cmd}
-
-

microservices/trader/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.12-slim
+
+COPY requirements.txt /
+
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+
+RUN mkdir -p /logs /config
+
+SHELL ["/bin/bash", "-c"]
+
+ENV PYTHONPATH=/
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

microservices/trader/entrypoint.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+LogDir=/logs
+
+Cmd="python3.12"
+Cmd+=" cvttpy/apps/microservices/trader/trader.py"
+
+Cmd+=" --config=${CONFIG_FILE:-/config/cvtt_musvc.cfg}"
+Cmd+=" --log_file=${LogDir}/${TRDR_LOG_FILE}"
+
+Cmd+=" ${TRDR_ADD_ARGS}"
+echo ${Cmd}
+exec ${Cmd}
+

microservices/trader/requirements.txt

@@ -0,0 +1,9 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5
+hjson>=3.1.0
+sortedcontainers>=2.4.0
+redis>=5.0.8
+python-dateutil>=2.8.2
+types-python-dateutil>=2.8.19.6
+cryptography>=43.0.0
+PyJWT>=2.10.1

release_version.txt

@@ -1 +0,0 @@
-0.0.9,homestore ==> hs01

research/crypto_exch_stats/Dockerfile

@@ -1,7 +1,9 @@
 FROM python:3.12-slim
 
+ARG FROM_DIR=./docker_dev/research/crypto_exch_stats
+
 WORKDIR /
-COPY docker_dev/research/crypto_exch_stats/requirements.txt /
+COPY ${FROM_DIR}/requirements.txt /
 
 RUN pip install --upgrade pip --root-user-action=ignore
 RUN pip install -r /requirements.txt --root-user-action=ignore
@@ -16,7 +18,7 @@ ENV PYTHONPATH=/
 CMD [                                                                      \
     "python3.12",                                                          \
     "cvttpy/apps//research/exchange_trading_stats.py",                     \
-    "--config=http://cloud23.cvtt.vpn:6789/apps/tests/listen_market_data", \
+    "--config=http://cloud16.cvtt.vpn:6789/apps/tests/listen_market_data", \
     "--active_exchanges=OKX,GEMINI,BITSTAMP,COINBASE_AT,BNBSPOT,KRAKEN",   \
     "--instruments=OKX:PAIR-BTC-USDT,GEMINI:PAIR-BTC-USD,BITSTAMP:PAIR-BTC-USD,COINBASE:PAIR-BTC-USD,BNBSPOT:PAIR-BTC-USDT,KRAKEN:PAIR-BTC-USD", \
     "--db_file=/app/data/exchange_trading_stats.db",                       \

shared/known_hosts

@@ -1,6 +0,0 @@
-|1|yT6WmkWOPxXSQeNECVJIX7M3upw=|wQPqbw6mXxwfO83sWVH9IRapSe0= ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNIOY8dNaWBcUu31F2OOuXUFGam2sx3Admn51SZR37MOMDEeKzpX+w0OnDGME9uOfuBToDW5F8ZoC+/p492uywZb4ZvUqVE8ic3uTaMro7zB9sykt4AO2rifM/0k7Fo1WoNWSaQptS3qyf7Umgl+qD5H8PxEM1OqKQl38BI5hfKUGx/WSYawgGIKkyO1eFwdjmSJ4RgQpN6/p5+1wJqbqbzkk0cEvZ06BbUbY+nUSUYY5WKa8ICBI0GKDdgRg9FXTV18r0YPT62HhShIyScbmg2h9hDKb7q9lLNLs2qg11jpSZCsIYTR2yms60uqXXq+isReLZ+W4z0t/8YVoID4Q82UlJ8R/LEqiJd8VcjZHaRVqtr7oVQjSurg1ChgCnOHHjvjZLVUg7QRMbCVT2/6PMAjjTyHUx8TKscYjz21ctPMn63pH/6lYzncQrt7ZT6PromVtnBs7p8dHaCyv18z9EBvGuE2rYOsfsCuR7FNlno7ej/gbe6CoaT2KwcNU8vpc=
-|1|QgYdyhwM1uy97fUH/l9jXfRnYlE=|P575Q9H/V0Hoszk+G9As1GozQmE= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHlFA8BawdNbgYsFPPUJiG7xfuWI6dwNIFkuUwqco+TPXanMG4CvZ0TjRgb2fwdMXAGyoNAmwcr8eR5HU8wTg8g=
-|1|vSzhghnKpUIGuOOgELHWGrbMxd8=|GY2Acsr6iqo4Y/dDS8V5glZ7tmA= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKz0QyrgFpp9NsPvZVwkjoua57SztFOMUnEzOLfW7kPr
-|1|LJg/h/d7KSfFYzdtrrEFpcpnF/4=|xRkJEoqiptYGTTt5EuQo4f3zo6c= ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKnC41A86Q2mVQWyRXHKzht6t8J1JJsRbYrQRpcMzh6VkVmOPDdAtl4P5iOH/GUaHr+HNhS7Y/QZS0yyjv8u1yOjb6ovfKvIuysj7f87wI1UOtQaUgX/PG5wupk/SgiUcfkpG+hg5wUACbsbMjp+HbE/OB7Nk5ZzUHNZcwQ55mRis8AzLW3IuFEboE+tJCxqTz8mr95ecqEWXdG4cJdDp7BMuCzsPCsOWgpcJiH+kgimok9n4qJPgoFSTVvHKBj0uJlm33rc8ns/Uu8knq6QfeSlzlN7uW0Xue6irW4Uaz69NbSUcFCeG83WNGO7tceYc7tmVM+9jjAtyhh9x1kGyk6thBMimRE5taa+5+bZP5tMctFsze6LVUGOothYW2MVgllJ+ovNFV/lin1y1591eAl340uSs4E4AJH6F/2Tta9OVGvNaXGMTYzKOLBYwo9sXJRHFuPjFxEs0Ya4pFUoCa8B53reNiOafrtwRU8/8G6zMgrF5z4FhCYBcIzoJdWCE=
-|1|9oemW7STLSSF7uf+SBwD7JWe0K8=|YCblOLOrxHLzbHkMuZfiUP8nkTU= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPEpfGoofmKWz3/t3Svaahh5ieq4LJ6/n+y3M0rOK/RgndUCESaTZXbw701NOF8BPcJczTQTuQPO2AJwrwQYcK0=
-|1|EC/isVVKdC1w/IthdEuhT29iU+w=|VClak/HfuZVXh4zlhhtx9euaTT0= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA2qvtcS1snKAdWKJefMCuKKgVlz0haOT0sElceM06qV

test/rsync_test/Dockerfile

@@ -0,0 +1,21 @@
+FROM python:3.12-slim
+
+ARG FROM_DIR=docker_dev/test/rsync_test
+
+COPY ${FROM_DIR}/test.sh /test.sh
+
+RUN apt-get update && apt-get install -y rsync openssh-client
+
+COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
+COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
+
+RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub
+
+# Shared Volumes
+RUN chmod +x /test.sh
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENV PYTHONPATH=/
+ENTRYPOINT [ "/test.sh" ]
+

test/rsync_test/test.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+date > /testfile.txt
+
+Source=/testfile.txt
+Targets=
+Targets="${Targets} cvtt@cloud21.cvtt.vpn:/tmp/"
+Targets="${Targets} cvtt@hs01.cvtt.vpn:/tmp/"
+
+
+for tgt in ${Targets}
+do
+    Cmd="/usr/bin/rsync -ahv"
+    Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
+    Cmd+=" ${Source} ${tgt}"
+    echo $Cmd
+    eval $Cmd
+done
+

trading.__DEPRECATED__/executor/Dockerfile

@@ -1,24 +0,0 @@
-FROM python:3.12-slim
-
-
-ARG FROM_DIR=docker_dev/trading/executor
-
-COPY ${FROM_DIR}/requirements.txt /
-RUN pip install --upgrade pip --root-user-action=ignore
-RUN pip install -r /requirements.txt --root-user-action=ignore
-
-COPY cvttpy /cvttpy
-COPY ${FROM_DIR}/.creds  /.creds
-
-
-COPY ${FROM_DIR}/run_executor.sh /run_executor.sh
-RUN chmod +x /run_executor.sh
-
-RUN mkdir /logs
-
-WORKDIR /
-SHELL ["/bin/bash", "-c"]
-ENV PYTHONPATH=/
-ENTRYPOINT [ "/run_executor.sh" ]
-
-

trading.__DEPRECATED__/executor/requirements.txt

@@ -1,6 +0,0 @@
-aiohttp>=3.7.4.post0
-redis>=5.0.8
-nest-asyncio>=1.5.5
-hjson>=3.1.0
-sortedcontainers>=2.4.0
-pandas>=1.5.3

trading.__DEPRECATED__/executor/run_executor.sh

@@ -1,53 +0,0 @@
-#!/bin/bash
-
-# runs in container
-
-
-# runs on host to start container
-usage() {
-    echo -n "Usage: $0"
-    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
-    echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
-    echo -n " [-a <admin_port (7222)>"]
-    echo -n " [-z (compress log)"]
-    echo
-    exit 1
-}
-
-Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
-ActiveExchanges=ALPACA_SNDBX
-AdminPort=7222
-COMPRESS_LOG=false
-
-while getopts "c:e:a:z" opt; do
-    case ${opt} in
-        c ) Config=$OPTARG  ;;
-        e ) ActiveExchanges=$OPTARG ;;
-        a ) AdminPort=$OPTARG ;;
-        z ) COMPRESS_LOG=true ;;
-        \? )
-            echo "Invalid option: -$OPTARG" >&2
-            usage
-            ;;
-        : )
-            echo "Option -$OPTARG requires an argument." >&2
-            usage
-            ;;
-    esac
-done
-
-Cmd="python3.12"
-Cmd="${Cmd} /cvttpy/apps/executor_app.py"
-Cmd="${Cmd} --config=${Config}"
-Cmd="${Cmd} --credentials_file=/.creds"
-# Cmd="${Cmd} --allow_dynamic_exch_inst"
-Cmd="${Cmd} --active_exchanges=${ActiveExchanges}"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_file=/logs/%T.EXECUTOR_APP.log"
-if ${COMPRESS_LOG}; then
-    Cmd="${Cmd} --compress_log"
-fi
-echo ${Cmd}
-eval ${Cmd}
-
-

trading.__DEPRECATED__/md_portal/Dockerfile

@@ -1,23 +0,0 @@
-FROM python:3.12-slim
-
-ARG FROM_DIR=docker_dev/trading/md_portal
-
-COPY ${FROM_DIR}/requirements.txt /
-RUN pip install --upgrade pip --root-user-action=ignore
-RUN pip install -r /requirements.txt --root-user-action=ignore
-
-COPY cvttpy /cvttpy
-COPY ${FROM_DIR}/.creds  /.creds
-
-
-COPY ${FROM_DIR}/run_md_portal.sh /run_md_portal.sh
-RUN chmod +x /run_md_portal.sh
-
-RUN mkdir /logs
-
-
-WORKDIR /
-SHELL ["/bin/bash", "-c"]
-ENV PYTHONPATH=/
-ENTRYPOINT [ "/run_md_portal.sh" ]
-

trading.__DEPRECATED__/md_portal/run_md_portal.sh

@@ -1,56 +0,0 @@
-#!/bin/bash
-
-# runs in container
-
-# runs on host to start container
-usage() {
-    echo -n "Usage: ${0}"
-    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
-    echo -n " [-e <active_exchanges (ALPACA_SNBOX)>]"
-    echo -n " [-a <admin_port (7220)>]"
-    echo -n " [-n <portal_name (MD_PORTAL_ALPACA)>]"
-    echo -n " [-z (compress log)"]
-    echo
-    exit 1
-}
-
-Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
-ActiveExchanges=ALPACA_SNDBX
-PortalName=MD_PORTAL_ALPACA_SNDBX
-AdminPort=7220
-COMPRESS_LOG=false
-
-
-while getopts "c:e:a:n:z" opt; do
-    case ${opt} in
-        c ) Config=$OPTARG  ;;
-        e ) ActiveExchanges=$OPTARG ;;
-        a ) AdminPort=$OPTARG ;;
-        z ) COMPRESS_LOG=true ;;
-        n ) PortalName=$OPTARG ;;
-        \? )
-            echo "Invalid option: -$OPTARG" >&2
-            usage
-            ;;
-        : )
-            echo "Option -$OPTARG requires an argument." >&2
-            usage
-            ;;
-    esac
-done
-
-Cmd="python3.12"
-Cmd="${Cmd} cvttpy/apps/md/md_portal.py"
-Cmd="${Cmd} --config=${Config}"
-Cmd="${Cmd} --credentials_file=/.creds"
-Cmd="${Cmd} --active_exchanges=${ActiveExchanges}"
-Cmd="${Cmd} --portal_name=${PortalName}"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_file=/logs/%T.${PortalName}.log"
-if ${COMPRESS_LOG}; then
-    Cmd="${Cmd} --compress_log"
-fi
-echo ${Cmd}
-eval ${Cmd}
-
-

trading.__DEPRECATED__/quant/Dockerfile

@@ -1,24 +0,0 @@
-FROM python:3.12-slim
-
-
-ARG FROM_DIR=docker_dev/trading/quant
-
-COPY ${FROM_DIR}/requirements.txt /
-RUN pip install --upgrade pip --root-user-action=ignore
-RUN pip install -r /requirements.txt --root-user-action=ignore
-
-COPY cvttpy /cvttpy
-COPY ${FROM_DIR}/.creds  /.creds
-
-
-COPY ${FROM_DIR}/run_quant.sh /run_quant.sh
-RUN chmod +x /run_quant.sh
-
-RUN mkdir /logs
-
-WORKDIR /
-SHELL ["/bin/bash", "-c"]
-ENV PYTHONPATH=/
-ENTRYPOINT [ "/run_quant.sh" ]
-
-

trading.__DEPRECATED__/quant/requirements.txt

@@ -1,11 +0,0 @@
-aiohttp>=3.7.4.post0
-nest-asyncio>=1.5.5
-hjson>=3.1.0
-sortedcontainers>=2.4.0
-pandas>=1.5.3
-#
-ta>=0.10.2
-tables>=3.9.1
-psycopg>=3.2.1
-#
-redis>=5.0.8

trading.__DEPRECATED__/quant/run_quant.sh

@@ -1,66 +0,0 @@
-#!/bin/bash
-
-# runs on host to start container
-usage() {
-    echo -n "Usage: $0"
-    echo -n " -b <book>"
-
-    echo -n " -S <strategy> (dflt: TRDALGO_001)"
-
-    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
-    echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
-    echo -n " [-a <admin_port (7224)>"]
-    echo -n " [-z (compress log)"]
-    echo
-    exit 1
-}
-
-Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
-ActiveExchanges=ALPACA_SNDBX-MDPORTAL
-AdminPort=7224
-Strategy=DAILY_STOCK_001
-Book=""
-COMPRESS_LOG=false
-
-
-while getopts "b:c:e:a:S:z" opt; do
-    case ${opt} in
-        c ) Config=$OPTARG  ;;
-        e ) ActiveExchanges=$OPTARG ;;
-        a ) AdminPort=$OPTARG ;;
-        z ) COMPRESS_LOG=true ;;
-        b ) Book=$OPTARG ;;
-        S ) Strategy=$OPTARG ;;
-        \? )
-            echo "Invalid option: -$OPTARG" >&2
-            usage
-            ;;
-        : )
-            echo "Option -$OPTARG requires an argument." >&2
-            usage
-            ;;
-    esac
-done
-
-if [ "${Book}" == "" ]; then
-    echo "Book is missing"
-    usage
-fi
-
-Cmd="python3.12"
-Cmd="${Cmd} cvttpy/apps/quant_app.py"
-Cmd="${Cmd} --config=${Config}"
-Cmd="${Cmd} --credentials_file=/.creds"
-Cmd="${Cmd} --book=${Book}"
-Cmd="${Cmd} --strategy=${Strategy}"
-# Cmd="${Cmd} --allow_dynamic_exch_inst"
-Cmd="${Cmd} --active_exchanges=${ActiveExchanges}"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_file=/logs/%T.QUANT.${Book}.log"
-if ${COMPRESS_LOG}; then
-    Cmd="${Cmd} --compress_log"
-fi
-echo ${Cmd}
-eval ${Cmd}
-
-

trading.__DEPRECATED__/risk_mgr/Dockerfile

@@ -1,23 +0,0 @@
-FROM python:3.12-slim
-
-ARG FROM_DIR=docker_dev/trading/risk_mgr
-
-COPY ${FROM_DIR}/requirements.txt /
-RUN pip install --upgrade pip --root-user-action=ignore
-RUN pip install -r /requirements.txt --root-user-action=ignore
-
-COPY cvttpy /cvttpy
-COPY ${FROM_DIR}/.creds  /.creds
-
-
-COPY ${FROM_DIR}/run_risk_mgr.sh /run_risk_mgr.sh
-RUN chmod +x /run_risk_mgr.sh
-
-RUN mkdir /logs
-
-
-WORKDIR /
-SHELL ["/bin/bash", "-c"]
-ENV PYTHONPATH=/
-ENTRYPOINT [ "/run_risk_mgr.sh" ]
-

trading.__DEPRECATED__/risk_mgr/run_risk_mgr.sh

@@ -1,64 +0,0 @@
-#!/bin/bash
-
-# runs in container
-
-# runs on host to start container
-usage() {
-    echo -n "Usage: $0"
-    echo -n " -b <book>"
-    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
-    echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
-    echo -n " [-a <admin_port (7223)>"]
-    echo -n " [-z (compress log)"]
-    echo
-    exit 1
-}
-
-
-
-Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
-ActiveExchanges=ALPACA_SNDBX-MDPORTAL
-AdminPort=7223
-Book=""
-COMPRESS_LOG=false
-
-
-while getopts "b:c:e:a:z" opt; do
-    case ${opt} in
-        c ) Config=$OPTARG  ;;
-        e ) ActiveExchanges=$OPTARG ;;
-        a ) AdminPort=$OPTARG ;;
-        z ) COMPRESS_LOG=true ;;
-        b ) Book=$OPTARG ;;
-        \? )
-            echo "Invalid option: -$OPTARG" >&2
-            usage
-            ;;
-        : )
-            echo "Option -$OPTARG requires an argument." >&2
-            usage
-            ;;
-    esac
-done
-
-if [ "${Book}" == "" ]; then
-    echo "Book is missing"
-    usage
-fi
-
-Cmd="python3.12"
-Cmd="${Cmd} cvttpy/apps/risk_manager_app.py"
-Cmd="${Cmd} --config=${Config}"
-Cmd="${Cmd} --credentials_file=/.creds"
-Cmd="${Cmd} --book=${Book}"
-# Cmd="${Cmd} --allow_dynamic_exch_inst"
-Cmd="${Cmd} --active_exchanges=${ActiveExchanges}"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_file=/logs/%T.RISK_MGR.${Book}.log"
-if ${COMPRESS_LOG}; then
-    Cmd="${Cmd} --compress_log"
-fi
-echo ${Cmd}
-eval ${Cmd}
-
-

trading.__DEPRECATED__/service_checker/Dockerfile

@@ -1,26 +0,0 @@
-FROM python:3.12-slim
-
-ARG FROM_DIR=docker_dev/trading/service_checker
-
-RUN apt-get update && apt-get install -y telnet redis-tools
-
-
-COPY ${FROM_DIR}/requirements.txt /
-RUN pip install --upgrade pip --root-user-action=ignore
-RUN pip install -r /requirements.txt --root-user-action=ignore
-
-COPY cvttpy /cvttpy
-COPY ${FROM_DIR}/.creds  /.creds
-
-
-COPY ${FROM_DIR}/run_svc_checker.sh /run_svc_checker.sh
-RUN chmod +x /run_svc_checker.sh
-
-# RUN mkdir /logs
-
-WORKDIR /
-SHELL ["/bin/bash", "-c"]
-ENV PYTHONPATH=/
-ENTRYPOINT [ "/run_svc_checker.sh" ]
-# ENTRYPOINT [ "/usr/bin/bash" ]
-

trading.__DEPRECATED__/service_checker/run_svc_checker.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-
-# runs in container
-
-# runs on host to start container
-usage() {
-    echo -n "Usage: ${0}"
-    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
-    echo
-    exit 1
-}
-
-Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
-
-
-while getopts ":c:" opt; do
-    case ${opt} in
-        c )
-            Config=$OPTARG
-            ;;
-        \? )
-            echo "Invalid option: -$OPTARG" >&2
-            usage
-            ;;
-        : )
-            echo "Option -$OPTARG requires an argument." >&2
-            usage
-            ;;
-    esac
-done
-
-
-Cmd="python3.12"
-Cmd="${Cmd} cvttpy/apps/utils/services_checker.py"
-Cmd="${Cmd} --config=${Config}"
-Cmd="${Cmd} --credentials_file=/.creds"
-Cmd="${Cmd} --log_level=ERROR"
-
-echo ${Cmd}
-eval ${Cmd}
-

trading.__DEPRECATED__/trader/Dockerfile

@@ -1,24 +0,0 @@
-FROM python:3.12-slim
-
-
-ARG FROM_DIR=docker_dev/trading/trader
-
-COPY ${FROM_DIR}/requirements.txt /
-RUN pip install --upgrade pip --root-user-action=ignore
-RUN pip install -r /requirements.txt --root-user-action=ignore
-
-COPY cvttpy /cvttpy
-COPY ${FROM_DIR}/.creds  /.creds
-
-
-COPY ${FROM_DIR}/run_trader.sh /run_trader.sh
-RUN chmod +x /run_trader.sh
-
-RUN mkdir /logs
-
-WORKDIR /
-SHELL ["/bin/bash", "-c"]
-ENV PYTHONPATH=/
-ENTRYPOINT [ "/run_trader.sh" ]
-
-

trading.__DEPRECATED__/trader/requirements.txt

@@ -1,6 +0,0 @@
-aiohttp>=3.7.4.post0
-redis>=5.0.8
-nest-asyncio>=1.5.5
-hjson>=3.1.0
-sortedcontainers>=2.4.0
-pandas>=1.5.3

trading.__DEPRECATED__/trader/run_trader.sh

@@ -1,71 +0,0 @@
-#!/bin/bash
-
-# runs in container
-# --------------------------------------------------------------------------
-
-# runs on host to start container
-usage() {
-    echo -n "Usage: $0"
-    echo -n " -b <book>"
-
-    echo -n " -A <algo> (dflt: TRDALGO_001)"
-
-    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
-    echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
-    echo -n " [-a <admin_port (7223)>"]
-    echo -n " [-z (compress log)"]
-    echo
-    exit 1
-}
-
-
-
-Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
-ActiveExchanges=ALPACA_SNDBX-MDPORTAL
-AdminPort=7226
-Algo=TRDALGO_001
-Book=""
-COMPRESS_LOG=false
-
-
-while getopts "b:c:e:a:A:z" opt; do
-    case ${opt} in
-        c ) Config=$OPTARG  ;;
-        e ) ActiveExchanges=$OPTARG ;;
-        a ) AdminPort=$OPTARG ;;
-        z ) COMPRESS_LOG=true ;;
-        b ) Book=$OPTARG ;;
-        A ) Algo=$OPTARG ;;
-        \? )
-            echo "Invalid option: -$OPTARG" >&2
-            usage
-            ;;
-        : )
-            echo "Option -$OPTARG requires an argument." >&2
-            usage
-            ;;
-    esac
-done
-
-if [ "${Book}" == "" ]; then
-    echo "Book is missing"
-    usage
-fi
-
-Cmd="python3.12"
-Cmd="${Cmd} cvttpy/apps/trader_app.py"
-Cmd="${Cmd} --config=${Config}"
-Cmd="${Cmd} --credentials_file=/.creds"
-Cmd="${Cmd} --book=${Book}"
-Cmd="${Cmd} --algo=${Algo}"
-# Cmd="${Cmd} --allow_dynamic_exch_inst"
-Cmd="${Cmd} --active_exchanges=${ActiveExchanges}"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_file=/logs/%T.TRADER.${Book}.log"
-if ${COMPRESS_LOG}; then
-    Cmd="${Cmd} --compress_log"
-fi
-echo ${Cmd}
-eval ${Cmd}
-
-

trading.__DEPRECATED__/trading_recorder/Dockerfile

@@ -1,23 +0,0 @@
-FROM python:3.12-slim
-
-ARG FROM_DIR=docker_dev/trading/trading_recorder
-
-COPY ${FROM_DIR}/requirements.txt /
-RUN pip install --upgrade pip --root-user-action=ignore
-RUN pip install -r /requirements.txt --root-user-action=ignore
-
-COPY cvttpy /cvttpy
-COPY ${FROM_DIR}/.creds  /.creds
-
-
-COPY ${FROM_DIR}/run_trading_recorder.sh /run_trading_recorder.sh
-RUN chmod +x /run_trading_recorder.sh
-
-RUN mkdir /logs
-
-
-WORKDIR /
-SHELL ["/bin/bash", "-c"]
-ENV PYTHONPATH=/
-ENTRYPOINT [ "/run_trading_recorder.sh" ]
-

trading.__DEPRECATED__/trading_recorder/run_trading_recorder.sh

@@ -1,60 +0,0 @@
-#!/bin/bash
-
-
-
-# runs on host to start container
-usage() {
-    echo -n "Usage: $0"
-    echo -n " -b <book>"
-    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
-    echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
-    echo -n " [-a <admin_port (7223)>"]
-    echo -n " [-z (compress log)"]
-    echo
-    exit 1
-}
-
-ActiveExchanges=ALPACA_SNDBX-MDPORTAL
-AdminPort=7225
-Book=""
-Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
-COMPRESS_LOG=false
-
-
-while getopts "b:c:e:a:z" opt; do
-    case ${opt} in
-        c ) Config=$OPTARG  ;;
-        e ) ActiveExchanges=$OPTARG ;;
-        a ) AdminPort=$OPTARG ;;
-        z ) COMPRESS_LOG=true ;;
-        b ) Book=$OPTARG ;;
-        \? )
-            echo "Invalid option: -$OPTARG" >&2
-            usage
-            ;;
-        : )
-            echo "Option -$OPTARG requires an argument." >&2
-            usage
-            ;;
-    esac
-done
-
-if [ "${Book}" == "" ]; then
-    echo "Book is missing"
-    usage
-fi
-
-Cmd="python3.12"
-Cmd="${Cmd} cvttpy/apps/trade_recorder.py"
-Cmd="${Cmd} --config=${Config}"
-Cmd="${Cmd} --credentials_file=/.creds"
-Cmd="${Cmd} --book=${Book}"
-# Cmd="${Cmd} --allow_dynamic_exch_inst"
-Cmd="${Cmd} --active_exchanges=${ActiveExchanges}"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_file=/logs/%T.TRADING_RECORDER.${Book}.log"
-if ${COMPRESS_LOG}; then
-    Cmd="${Cmd} --compress_log"
-fi
-echo ${Cmd}
-eval ${Cmd}