fix

market_data/_OLD_/alpaca_md_day/.creds

@@ -1,7 +0,0 @@
-{
-    "__dummy__": null
-    , "ALPACA_SANDBOX": {
-        "api_key": "PKLZSLFZMFMN1R28K9HK"
-        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
-    }
-}

market_data/_OLD_/alpaca_md_day/Dockerfile

@@ -1,46 +0,0 @@
-FROM python:3.12-slim
-
-ARG FROM_DIR=docker_dev/market_data/alpaca_md_day
-
-# Update the package list and install required packages
-RUN apt-get update && apt-get install -y \
-    apt-utils \
-    libpq-dev \
-    build-essential
-RUN apt-get update && apt-get install -y rsync openssh-client
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    software-properties-common \
-    curl \
-    parallel \
-    jq
-
-RUN pip install --upgrade pip
-
-
-COPY ${FROM_DIR}/requirements.txt /
-RUN pip install -r /requirements.txt
-
-COPY cvttpy /cvttpy
-COPY ${FROM_DIR}/.creds  /.creds
-COPY ${FROM_DIR}/.creds  /root/.creds
-COPY ${FROM_DIR}/alpaca_md_day.sh /alpaca_md_day.sh
-
-COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
-COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
-COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
-
-RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
-
-
-# Shared Volumes
-RUN mkdir -p /app/data
-RUN mkdir /logs
-RUN chmod +x /alpaca_md_day.sh
-
-WORKDIR /
-SHELL ["/bin/bash", "-c"]
-ENV PYTHONPATH=/
-ENTRYPOINT [ "/alpaca_md_day.sh" ]
-
-# CMD [ "echo", "alpaca_md_day"]

market_data/_OLD_/alpaca_md_day/HOWTO.md

@@ -1,116 +0,0 @@
-# Build/Dev Host
-## Build
-```bash
-Version=$(cat /home/oleg/develop/cvtt2/cvttpy/release_version.txt)
-
-cd /home/oleg/develop/cvtt2
-docker build  -t crypto_md_day -t crypto_md_day:${Version} -f docker_dev/crypto_md_day/Dockerfile .
-
-## Deploy
-
-/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day
-
-```
-
-## On target Machine make sure user can use docker
-```bash
-sudo usermod -a -G docker cvtt
-```
-**Re-login as cvtt after adding cvtt to the group**
-
-### Clean
-```bash
-docker rm -f crypto_md_day
-docker images --filter=reference=crypto_md_day --format="{{.ID}}" | uniq | xargs docker rmi -f
-```
-### Test run locally
-```
-docker load -i /home/oleg/docker_images/crypto_md_day.img.tar
-mkdir -p /tmp/data
-mkdir -p /tmp/logs
-docker run --network="host" --name=crypto_md_day -v /tmp/data:/app/data -v /tmp/logs:/logs crypto_md_day
-```
-
-## Restarting
-```bash
-docker restart crypto_md_day
-```
-
-# Useful Commands
-```bash
-docker images
-docker ps -a
-docker image rm <image>
-docker rm <container>
-docker exec -it <container> /bin/bash
-```
-
----
-```bash
-docker rm -f crypto_md_day
-docker images --filter=reference=crypto_md_day --format="{{.ID}}" | uniq | xargs docker rmi -f
-
-/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day cryptoval4.cvtt.vpn
-docker tag crypto_md_day:latest cloud21.cvtt.vpn:5500/crypto_md_day:latest
-docker push cloud21.cvtt.vpn:5500/crypto_md_day:latest
-
-# Local test
-docker load -i /home/oleg/docker_images/crypto_md_day.img.tar  && docker run --network="host" --name=crypto_md_day -v /tmp/data:/app/data crypto_md_day -h cvttdata -d 20240717 -s coinbase
-```
-
-## Local (cloud21) docker registry
-### Run registry container (cloud21)
-
-
-```bash
-mkdir /opt/store/cvtt/docker_registry/data
-vi /opt/store/cvtt/docker_registry/config.yml
-```
-
-```yaml
-version: 0.1
-log:
-  fields:
-    service: registry
-storage:
-  filesystem:
-    rootdirectory: /var/lib/registry
-http:
-  addr: :5500
-
-```
-```bash
-docker run -d -p 5500:5500 --name registry -v /opt/store/cvtt/docker_registry/config.yml:/etc/docker/registry/config.yml -v /opt/store/cvtt/docker_registry/data:/var/lib/registry registry:2
-```
-
-### Building
-#### /etc/docker/daemon.json
-```json
-{
-  "insecure-registries": ["cloud21.cvtt.vpn:5500"]
-}
-```
-```bash
-sudo systemctl restart docker
-```
-```bash
-/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day cryptoval4.cvtt.vpn
-docker tag crypto_md_day:latest cloud21.cvtt.vpn:5500/crypto_md_day:latest
-docker push cloud21.cvtt.vpn:5500/crypto_md_day:latest
-```
-
-### Loading and Running (cryptoval4)
-#### /etc/docker/daemon.json
-```json
-{
-  "insecure-registries": ["cloud21.cvtt.vpn:5500"]
-}
-```
-```bash
-sudo systemctl restart docker
-```
-```
-docker pull cloud21.cvtt.vpn:5500/crypto_md_day:latest
-docker run --network="host" --name=cloud21.cvtt.vpn:5500/crypto_md_day crypto_md_day -h cvttdata -d 20240717 -s coinbase
-```
-

market_data/_OLD_/alpaca_md_day/alpaca_md_day.sh

@@ -1,138 +0,0 @@
-#!/bin/bash
-
-# --- Settings
-export PYTHONPATH=/
-export Python=python3.12
-export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
-export PyScript=/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
-
-export OutputDir=/app/data/alpaca_md # Local
-export LogDir=/logs
-
-DEFAULT_NUM_JOBS=10
-SLEEP_SEC_AFTER_PROC=2
-DEFAULT_SLICE_SIZE=500
-# --- Settings
-
-mkdir -p ${OutputDir}
-mkdir -p ${LogDir}
-
-
-usage() {
-  echo "Usage: $0 <date (YYYY-MM-DD)> [<num-jobs> (30) ] [<instrument_list_file>] "
-  exit 1
-}
-
-echo "CommandLine: ${*}"
-
-Start=${1}
-NumJobs=${2}
-
-export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
-
-is_business_day() {
-    dt=${1}
-
-    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
-    if [ -n  "${open_time}" ]; then
-        return 0
-    else
-        return 1
-    fi
-
-}
-export -f is_business_day
-
-
-if [ -z ${Start}  ] ; then
-    echo "start is not specified, getting yesterday..."
-    Start=$(date -d "yesterday" "+%Y-%m-%d")
-    echo "Start=${Start}"
-fi
-
-while true; do
-    if is_business_day ${Start}; then
-        break
-    fi
-    echo "${Start} is not business day in US"
-    Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
-done
-
-End=${Start}   # only 1 day is practical
-
-if [ "${NumJobs}" == "" ] ; then
-    NumJobs=${DEFAULT_NUM_JOBS}
-fi
-
-echo "Start=${Start} End=${End} NumJobs=${NumJobs}"
-
-
-run_proc() {
-    Inst=${1}
-    Start=${2}
-    End=${3}
-    echo "Running for $Inst"
-
-    Cmd="${Python} ${PyScript}"
-    Cmd="${Cmd} --config=${Config}"
-    Cmd="${Cmd} --output_dir=${OutputDir}"
-    Cmd="${Cmd} --instruments=ALPACA:${Inst}"
-    Cmd="${Cmd} --start=${Start}"
-    if [ "${End}" != "" ]; then
-        Cmd="${Cmd} --end=${End}"
-    fi
-    Cmd="${Cmd} --log_file=${LogDir}/${Inst}.log"
-    Cmd="${Cmd} --log_level=WARNING"
-    echo ${Cmd}
-    eval ${Cmd}
-
-    sleep 2 #${SLEEP_SEC_AFTER_PROC}
-}
-
-export -f run_proc
-
-key=$(jq -r '.["ALPACA_SANDBOX"] | .api_key' ~/.creds)
-secret=$(jq -r '.["ALPACA_SANDBOX"] | .secret_key' ~/.creds)
-
-Cmd="curl -s --request GET --url 'https://paper-api.alpaca.markets/v2/assets?status=active'"
-Cmd="${Cmd} --header 'APCA-API-KEY-ID: ${key}'"
-Cmd="${Cmd} --header 'APCA-API-SECRET-KEY: ${secret}'"
-Cmd="${Cmd} --header 'accept: application/json'"
-Cmd="${Cmd} | jq '.[] | select(.class == \"us_equity\" and .exchange != \"OTC\") | .symbol'"
-Cmd="${Cmd}  | sed 's/\"//g'"
-Cmd="${Cmd} | sed 's/^/STOCK-/'"
-# split string into array
-Instruments=()
-for Inst in $(eval ${Cmd})
-do
-    Instruments+=("$Inst")
-done
-
-slice_size=${DEFAULT_SLICE_SIZE} # 10K symbols parallel cannot handle
-for ((ii=0; ii <${#Instruments[@]}; ii+=slice_size)); do
-    InstSlice=("${Instruments[@]:ii:slice_size}")
-
-    parallel -j ${NumJobs} run_proc {} ${Start} ${End} ::: "${InstSlice[@]}"
-done
-
-echo "Compressing"
-for file in $(find ${OutputDir} -type f -name '*db' -print )
-do
-    echo "Compressing ${file}"
-    gzip ${file}
-done
-
-
-Source=/app/data/
-Targets=
-Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity"
-Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
-
-
-for tgt in ${Targets}
-do
-    Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
-    echo $Cmd
-    eval $Cmd
-done
-

market_data/alpaca_hbar/.creds

@@ -1,7 +0,0 @@
-{
-    "__dummy__": null
-    , "ALPACA_SANDBOX": {
-        "api_key": "PKLZSLFZMFMN1R28K9HK"
-        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
-    }
-}

market_data/alpaca_hbar/Dockerfile

@@ -15,7 +15,7 @@ RUN apt-get update && apt-get install -y \
 # must be from disted version
 COPY cvtt-rust/bin/alpaca_md_hbar_loader /
 
-COPY ${FROM_DIR}/.creds  /.creds
+# COPY ${FROM_DIR}/.creds  /.creds
 COPY ${FROM_DIR}/alpaca_hbar.sh /alpaca_hbar.sh
 
 COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa

market_data/alpaca_hbar/alpaca_hbar.sh

@@ -43,10 +43,6 @@ while getopts ":d:N:" opt; do
   esac
 done
 
-# if [ -z ${Symbols}  ] ; then
-#     echo "symbols are not specified"
-#     usage
-# fi
 
 mkdir -p ${OutputDir}
 mkdir -p ${LogDir}

market_data/alpaca_qat/.creds

@@ -1,7 +0,0 @@
-{
-    "__dummy__": null
-    , "ALPACA_SANDBOX": {
-        "api_key": "PKLZSLFZMFMN1R28K9HK"
-        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
-    }
-}

market_data/alpaca_qat/Dockerfile

@@ -16,7 +16,7 @@ RUN apt-get update && apt-get install -y \
 # must be from disted version
 COPY cvtt-rust/bin/alpaca_md_qat_loader /
 
-COPY ${FROM_DIR}/.creds  /.creds
+# COPY ${FROM_DIR}/.creds  /.creds
 COPY ${FROM_DIR}/alpaca_qat.sh /alpaca_qat.sh
 
 COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa

market_data/crypto_md_day/.creds

@@ -1,17 +0,0 @@
-{
-    "__dummy__": null
-     , "TSDB_MD_CLD21_RO": {
-         "host": "cloud21.cvtt.vpn"
-         , "port": 5432
-         , "user": "cvtt_ro"
-         , "database": "cvtt_md"
-         , "password": "3Gkotqa0ZSmZLXSH3RkTUm"
-     }
-    , "TSDB_MD_CVTTDATA_RO": {
-        "host": "cvttdata.cvtt.vpn"
-        , "port": 5432
-        , "user": "cvtt_ro"
-        , "database": "cvtt_md"
-        , "password": "3Gkotqa0ZSmZLXSH3RkTUm"
-    }
-}

market_data/crypto_md_day/Dockerfile

@@ -6,7 +6,7 @@ COPY ${FROM_DIR}/requirements.txt /
 
 
 COPY cvttpy /cvttpy
-COPY ${FROM_DIR}/.creds  /.creds
+# COPY ${FROM_DIR}/.creds  /.creds
 COPY ${FROM_DIR}/crypto_md_day.sh /crypto_md_day.sh
 
 RUN pip install --upgrade pip --root-user-action=ignore

market_data/crypto_md_day/crypto_md_day.sh

@@ -52,7 +52,7 @@ export PYTHONPATH=/
 
 Cmd="python3.12"
 Cmd="${Cmd} cvttpy/research/utils/archive_ts_md.py"
-Cmd="${Cmd} --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder"
+Cmd="${Cmd} --config=http://cloud23.cvtt.vpn:6789/apps/md_recorder"
 Cmd="${Cmd} --db_credentials_key=${CredKey}"
 Cmd="${Cmd} --credentials_file=/.creds"
 Cmd="${Cmd} --date=${Date}"

market_data/md_recorder/Dockerfile

@@ -12,7 +12,7 @@ RUN pip install --upgrade pip --root-user-action=ignore
 RUN pip install -r /requirements.txt --root-user-action=ignore
 
 COPY cvttpy /cvttpy
-COPY ${FROM_DIR}/.creds  /.creds
+# COPY ${FROM_DIR}/.creds  /.creds
 COPY ${FROM_DIR}/run_md_recorder.sh /run_md_recorder.sh
 
 

market_data/md_recorder/run_md_recorder.sh

@@ -26,7 +26,6 @@ if  ! is_valid "${job}"; then
     usage
 fi
 
-# ConfigServer=cloud16.cvtt.vpn
 ConfigServer=cloud23.cvtt.vpn:6789
 
 Cmd="python3.12"

market_data/md_recorder_monitor/Dockerfile

@@ -13,7 +13,7 @@ RUN pip install --upgrade pip --root-user-action=ignore
 RUN pip install -r /requirements.txt --root-user-action=ignore
 
 COPY cvttpy /cvttpy
-COPY ${FROM_DIR}/.creds  /.creds
+# COPY ${FROM_DIR}/.creds  /.creds
 COPY ${FROM_DIR}/run_mdr_monitor.sh /run_mdr_monitor.sh
 
 

market_data/md_recorder_monitor/run_mdr_monitor.sh

@@ -32,7 +32,6 @@ if [ "${2}" != "" ]; then
 fi
 
 
-# ConfigServer=cloud16.cvtt.vpn
 ConfigServer=cloud23.cvtt.vpn:6789
 
 Cmd="python3.12"

microservices/docker-compose.yml

@@ -0,0 +1,52 @@
+version: "3.9"
+
+services:
+    md_gateway:
+        image: cloud21.cvtt.vpn:5500/md_gateway:latest
+        container_name: md_gateway
+        environment:
+          - REDIS_HOST=redis
+        volumes:
+          - ./.creds:/.creds
+          - ./config:/config
+          - ./logs:/logs
+          - ./data:/data
+        depends_on:
+          - redis
+
+    tester:
+        image: cloud21.cvtt.vpn:5500/tester:latest
+        container_name: tester
+        environment:
+          - REDIS_HOST=redis
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+          - ./data:/data
+        depends_on:
+          - redis
+          - md_gateway
+
+    redis:
+        image: redis:latest
+        container_name: redis
+        ports:
+          - "6379:6379"
+        volumes:
+          - ./data/redis:/data
+
+
+#   test_program:
+#     image: your_test_program_image:latest
+#     container_name: test_program
+#     environment:
+#       - REDIS_HOST=redis
+#     volumes:
+#       - ./config:/shared/config
+#       - ./logs:/shared/logs
+#     depends_on:
+#       - market_data_gateway
+#       - redis
+
+# volumes:
+#   redis_data:

microservices/md_gateway/Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.12-slim
+
+ARG ROOT_MCRSVC_DIR=docker_dev/microservices
+ARG FROM_DIR=${ROOT_MCRSVC_DIR}/md_gateway
+
+COPY ${ROOT_MCRSVC_DIR}/requirements.txt /
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+# COPY ${ROOT_MCRSVC_DIR}/.creds  /.creds
+
+
+COPY ${FROM_DIR}/run_mdgw.sh /run_mdgw.sh
+RUN chmod +x /run_mdgw.sh
+
+RUN mkdir /logs
+RUN mkdir -p /config
+
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENV PYTHONPATH=/
+ENTRYPOINT [ "/run_mdgw.sh" ]
+

microservices/md_gateway/run_mdgw.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# runs in container
+
+# runs on host to start container
+usage() {
+    echo -n "Usage: ${0}"
+    echo -n " [-c <config (/config/cvtt.cfg)>]"
+    echo -n " [-a <admin_port (7220)>]"
+    echo -n " [-z (compress log)"]
+    echo
+    exit 1
+}
+
+Config=/config/cvtt.cfg
+AdminPort=7220
+COMPRESS_LOG=false
+
+
+while getopts "c:a:z" opt; do
+    case ${opt} in
+        c ) Config=$OPTARG  ;;
+        a ) AdminPort=$OPTARG ;;
+        z ) COMPRESS_LOG=true ;;
+        \? )
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            ;;
+        : )
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            ;;
+    esac
+done
+
+Cmd="python3.12"
+Cmd="${Cmd} cvttpy/apps/microservices/market_gateways/md_gateway.py"
+Cmd="${Cmd} --config=${Config}"
+Cmd="${Cmd} --credentials_file=/.creds"
+Cmd="${Cmd} --admin_port=${AdminPort}"
+Cmd="${Cmd} --log_file=/logs/%T.md_gateway.log"
+if ${COMPRESS_LOG} ; then
+    Cmd="${Cmd} --compress_log"
+fi
+echo ${Cmd}
+eval ${Cmd}
+
+

microservices/oe_gateway/.dockerignore

@@ -0,0 +1,3 @@
+.git
+**/__pycache__
+.pipenv

microservices/oe_gateway/Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.12-slim
+
+ARG ROOT_MCRSVC_DIR=docker_dev/microservices
+ARG FROM_DIR=${ROOT_MCRSVC_DIR}/oe_gateway
+
+COPY ${ROOT_MCRSVC_DIR}/requirements.txt /
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+COPY ${ROOT_MCRSVC_DIR}/.creds  /.creds
+
+
+COPY ${FROM_DIR}/run_oegw.sh /run_oegw.sh
+RUN chmod +x /run_oegw.sh
+
+RUN mkdir /logs
+RUN mkdir -p /config
+
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENV PYTHONPATH=/
+ENTRYPOINT [ "/run_oegw.sh" ]
+

microservices/oe_gateway/run_oegw.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# runs in container
+
+# runs on host to start container
+usage() {
+    echo -n "Usage: ${0}"
+    echo -n " [-c <config (/config/cvtt.cfg)>]"
+    echo -n " [-a <admin_port (7220)>]"
+    echo -n " [-z (compress log)"]
+    echo
+    exit 1
+}
+
+Config=/config/cvtt.cfg
+AdminPort=7220
+COMPRESS_LOG=false
+
+
+while getopts "c:a:z" opt; do
+    case ${opt} in
+        c ) Config=$OPTARG  ;;
+        a ) AdminPort=$OPTARG ;;
+        z ) COMPRESS_LOG=true ;;
+        \? )
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            ;;
+        : )
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            ;;
+    esac
+done
+
+Cmd="python3.12"
+Cmd="${Cmd} cvttpy/apps/microservices/market_gateways/order_entry_gateway.py"
+Cmd="${Cmd} --config=${Config}"
+Cmd="${Cmd} --credentials_file=/.creds"
+Cmd="${Cmd} --admin_port=${AdminPort}"
+Cmd="${Cmd} --log_file=/logs/%T.md_gateway.log"
+if ${COMPRESS_LOG} ; then
+    Cmd="${Cmd} --compress_log"
+fi
+echo ${Cmd}
+eval ${Cmd}
+
+

microservices/requirements.txt

@@ -1,7 +1,7 @@
 aiohttp>=3.7.4.post0
 nest-asyncio>=1.5.5
-psycopg>=3.2.1
 hjson>=3.1.0
-pandas>=1.5.3
 sortedcontainers>=2.4.0
 redis>=5.0.8
+python-dateutil>=2.8.2
+types-python-dateutil>=2.8.19.6

microservices/tester/.dockerignore

@@ -0,0 +1,3 @@
+.git
+**/__pycache__
+.pipenv

microservices/tester/Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3.12-slim
+
+ARG ROOT_MUSVC_DIR=docker_dev/microservices
+ARG FROM_DIR=${ROOT_MUSVC_DIR}/tester
+
+COPY ${ROOT_MUSVC_DIR}/requirements.txt /
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+# COPY ${ROOT_MUSVC_DIR}/.creds  /.creds
+
+
+COPY ${FROM_DIR}/run_tester.sh /run_tester.sh
+RUN chmod +x /run_tester.sh
+
+RUN mkdir /logs
+RUN mkdir -p /config
+RUN mkdir -p /data
+
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENV PYTHONPATH=/
+ENTRYPOINT [ "/run_tester.sh" ]
+

microservices/tester/run_tester.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# runs in container
+
+# runs on host to start container
+usage() {
+    echo -n "Usage: ${0}"
+    echo -n " [-c <config (/config/cvtt.cfg) >]"
+    echo -n " [-i <instruments (COINBASE:PAIR-BTC-USD,BNBFUT:PERP-BTC-USDT) >]"
+    echo -n " [-d <data types (TRADES) >]"
+    echo -n " [-a <admin_port (7221) >]"
+    echo -n " [-z (compress log (false))"]
+    echo
+    exit 1
+}
+
+Config=/config/cvtt.cfg
+AdminPort=7221
+COMPRESS_LOG=false
+Instruments="COINBASE:PAIR-BTC-USD,BNBFUT:PERP-BTC-USDT"
+DataTypes="TRADES"
+
+while getopts "a:c:d:i:z" opt; do
+    case ${opt} in
+        c ) Config=$OPTARG  ;;
+        a ) AdminPort=$OPTARG ;;
+        i ) Instruments=$OPTARG ;;
+        d ) DataTypes=$OPTARG ;;
+        z ) COMPRESS_LOG=true ;;
+        \? )
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            ;;
+        : )
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            ;;
+    esac
+done
+
+Cmd="python3.12"
+Cmd="${Cmd} cvttpy/apps/microservices/tester/musvc_tester.py"
+Cmd="${Cmd} --config=${Config}"
+Cmd="${Cmd} --credentials_file=/.creds"
+Cmd="${Cmd} --admin_port=${AdminPort}"
+Cmd="${Cmd} --data_types=${DataTypes}"
+Cmd="${Cmd} --instruments=${Instruments}"
+Cmd="${Cmd} --log_file=/logs/%T.tester.log"
+if ${COMPRESS_LOG} ; then
+    Cmd="${Cmd} --compress_log"
+fi
+echo ${Cmd}
+eval ${Cmd}
+
+

release_version.txt

@@ -1 +1 @@
-0.0.4,crypto_exch_stats
+0.0.7.F1,creds to be taken from volumes

tools/code-server-plus/Dockerfile

@@ -0,0 +1,49 @@
+FROM codercom/code-server:latest
+
+ARG FROM_DIR=./
+USER root
+# Update the package list and install required packages
+RUN sudo apt-get update && apt-get install -y \
+    apt-utils \
+    rsync \
+    openssh-client \
+    software-properties-common \ 
+    build-essential \
+    zlib1g-dev \
+    libncurses5-dev \
+    libgdbm-dev \ 
+    libnss3-dev \
+    libssl-dev \
+    libreadline-dev \
+    libffi-dev \
+    libsqlite3-dev \
+    libbz2-dev \
+    curl \
+    wget \
+    jq
+
+# ----------------------------------------
+RUN mkdir -p /tmp
+WORKDIR /tmp
+RUN wget https://www.python.org/ftp/python/3.12.5/Python-3.12.5.tgz
+
+RUN tar -xf Python-3.12.5.tgz
+WORKDIR /tmp/Python-3.12.5
+
+RUN ./configure --enable-optimizations --prefix=/usr
+RUN make -j$(nproc)
+RUN make altinstall
+
+# ----------------------------------------
+RUN rm /usr/bin/python3
+RUN ln -s /usr/bin/python3.12 /usr/bin/python3
+
+RUN ln -snf /usr/bin/pip3.12 /usr/bin/pip
+
+COPY ./requirements.txt /
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /home/coder

tools/code-server-plus/build.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+docker build -t code-server-plus -t code-server-plus:latest -f Dockerfile .

tools/code-server-plus/docker-compose.yml

@@ -0,0 +1,15 @@
+version: "3.8"
+services:
+  code-server:
+    container_name: code-server-plus
+    image: code-server-plus:latest
+    restart: unless-stopped
+    user: 1001:1001
+    volumes:
+      - /opt/code-server/.local:/home/coder/.local
+      - /opt/code-server/.config:/home/coder/.config
+      - /opt/code-server/projects:/home/coder/project
+    environment:
+      - DOCKER_USER=cvtt
+    ports:
+      - "8080:8080"

tools/code-server-plus/requirements.txt

@@ -0,0 +1,162 @@
+aiohttp>=3.7.4.post0
+appdirs>=1.4.4
+asn1crypto>=0.24.0
+astroid>=2.8.0
+async-timeout>=3.0.1
+attrs>=22.1.0
+Automat>=0.8.0
+backports.entry-points-selectable>=1.1.0
+bcrypt>=4.0.0
+black>=22.6.0
+blinker>=1.4
+boto3>=1.24.43
+botocore>=1.27.43
+cachetools>=4.1.1
+certifi>=2019.11.28
+cffi>=1.15.0
+chardet>=4.0.0
+charset-normalizer>=2.0.7
+ciso8601>=2.2.0
+click>=8.1.3
+coloredlogs>=15.0.1
+configobj>=5.0.6
+constantly>=15.1.0
+cryptography>=2.8
+dataclasses-json>=0.5.1
+distlib>=0.3.3
+distro>=1.4.0
+docker>=5.0.3
+entrypoints>=0.3
+eventkit>=0.8.9
+ExportCsvToInflux>=0.2.2
+filelock>=3.3.2
+google-api-core>=1.22.2
+google-api-python-client>=2.22.0
+google-auth>=1.21.1
+google-auth-httplib2>=0.1.0
+google-auth-oauthlib>=0.4.6
+googleapis-common-protos>=1.52.0
+greenlet>=1.1.2
+hiredis>=1.1.0
+hjson>=3.0.2
+httplib2>=0.18.1
+humanfriendly>=10.0
+hyperlink>=19.0.0
+ib-insync>=0.9.70
+ibapi
+idna>=2.8
+importlab>=0.7
+importlib-metadata>=1.5.0
+incremental>=16.10.1
+influxdb>=5.3.1
+influxdb-client>=1.29.1
+isort>=5.5.1
+jedi>=0.18.0
+Jinja2>=2.10.1
+jmespath>=1.0.1
+json-cfg>=0.4.2
+jsonpatch>=1.22
+jsonpointer>=2.0
+jsonschema>=3.2.0
+keyring>=18.0.1
+kwonly-args>=1.0.10
+launchpadlib>=1.10.13
+lazr.restfulclient>=0.14.2
+lazr.uri>=1.0.3
+lazy-object-proxy>=1.4.3
+libcst>=0.4.7
+lxml>=4.6.3
+lxml-stubs>=0.2.0
+MarkupSafe>=1.1.0
+marshmallow>=3.7.0
+marshmallow-enum>=1.5.1
+mccabe>=0.6.1
+more-itertools>=4.2.0
+msgpack>=0.6.1
+multidict>=4.7.6
+multitasking>=0.0.10
+mypy>=0.961
+mypy-extensions>=0.4.3
+nest-asyncio>=1.5.5
+networkx>=2.8.3
+ninja>=1.10.2.3
+numpy #>=1.19.1
+oauthlib>=3.1.0
+pandas>=1.3.3
+pandas-stubs>=1.2.0.17
+paramiko>=2.11.0
+parso>=0.8.1
+pathspec>=0.9.0
+pbr>=5.6.0
+pexpect>=4.6.0
+platformdirs>=2.3.0
+protobuf>=3.13.0
+ptyprocess>=0.7.0
+pyasn1>=0.4.2
+pyasn1-modules>=0.2.1
+pycparser>=2.20
+pydot>=1.4.2
+PyHamcrest>=1.9.0
+PyJWT>=1.7.1
+pylint>=2.11.1
+pymongo>=4.0.1
+PyNaCl>=1.5.0
+pynvim>=0.4.1
+pyOpenSSL>=19.0.0
+pyparsing>=3.0.9
+pyrsistent>=0.15.5
+pyserial>=3.4
+python-dateutil>=2.8.2
+python-rapidjson>=0.9.1
+pytz>=2020.1
+PyYAML>=5.3.1
+redis>=5.0.8
+regex>=2021.11.10
+requests>=2.27.1
+requests-oauthlib>=1.3.0
+requests-unixsocket>=0.2.0
+rsa>=4.6
+Rx>=3.2.0
+s3transfer>=0.6.0
+SecretStorage>=2.3.1
+service-identity>=18.1.0
+simplefix>=1.0.14
+simplejson>=3.16.0
+six>=1.14.0
+slackclient>=2.9.3
+sortedcontainers>=2.4.0
+SQLAlchemy>=1.4.39
+ssh-import-id>=5.10
+stringcase>=1.2.0
+tabulate>=0.8.10
+termcolor>=1.1.0
+testresources>=2.0.1
+toml>=0.10.2
+tomli>=2.0.1
+typed-ast>=1.5.1
+types-chardet>=4.0.1
+types-cryptography>=3.3.23
+types-paramiko>=2.11.6
+types-PyMySQL>=1.0.18
+types-python-dateutil>=2.8.0
+types-pytz>=2022.1.1
+types-redis>=4.2.6
+types-requests>=2.25.7
+types-simplejson>=3.17.0
+types-termcolor>=1.1.1
+typing-inspect>=0.6.0
+typing_extensions>=4.3.0
+uritemplate>=3.0.1
+urllib3>=1.26.9
+uvloop>=0.14.0
+virtualenv>=20.9.0
+wadllib>=1.3.3
+websocket-client>=0.53.0
+websockets>=8.1
+wrapt>=1.12.1
+yapf>=0.30.0
+yarl>=1.5.1
+yfinance>=0.1.70
+zipp>=1.0.0
+zope.interface>=5.4.0
+

trading/executor/.creds

@@ -1,12 +0,0 @@
-{
-    "__dummy__": null
-    , "ALPACA_SANDBOX": {
-        "api_key": "PKLZSLFZMFMN1R28K9HK"
-        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
-    }
-    , "MATTERMOST": {
-        "url": "https://mattermost.cryptovaltrading.com"
-        , "team": "CVTT"
-        , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
-    }
-}

trading/md_portal/.creds

@@ -1,7 +0,0 @@
-{
-    "__dummy__": null
-    , "ALPACA_SANDBOX": {
-        "api_key": "PKLZSLFZMFMN1R28K9HK"
-        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
-    }
-}

trading/quant/.creds

@@ -1,21 +0,0 @@
-{
-    "TSDB_MD_CLD19_RO": {
-        "host": "cloud19.cvtt.vpn"
-        , "port": 5432
-        , "user": "cvtt_ro"
-        , "database": "cvtt_md"
-        , "password": "3Gkotqa0ZSmZLXSH3RkTUm"
-    }
-    , "TSDB_MD_CVTTDATA_RO": {
-        "host": "cvttdata.cvtt.vpn"
-        , "port": 5432
-        , "user": "cvtt_ro"
-        , "database": "cvtt_md"
-        , "password": "3Gkotqa0ZSmZLXSH3RkTUm"
-    }
-    , "MATTERMOST": {
-        "url": "https://cloud11.cryptovaltrading.com"
-        , "team": "CVTT"
-        , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
-    }
-}

trading/risk_mgr/.creds

@@ -1,20 +0,0 @@
-{
-    "__dummy__": null
-    , "ALPACA_SANDBOX": {
-        "api_key": "PKLZSLFZMFMN1R28K9HK"
-        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
-    }
-    , "TSDB_TRD_CVTTDATA": {
-        "host": "cvttdata.cvtt.vpn"
-        , "port": "5432"
-        , "database": "cvtt_trading"
-        , "user": "cvtt"
-        , "password": "ICdIh0JnMM7vM7Pf"
-    }
-    , "COINBASE_ADV_RO": {
-        "api_key": "Ip4QpRtRHBmWNpc3"
-        , "secret_key": "N6ZEMdIdUYUldkJRTUcCRKUFot4Fp2j0"
-        , "account_uuid": "???"
-        , "book_id": "CVTT_BK01"
-   }
-}

trading/service_checker/.creds

@@ -1,8 +0,0 @@
-{
-    "__dummy__": null
-    , "MATTERMOST": {
-        "url": "https://mattermost.cryptovaltrading.com"
-        , "team": "CVTT"
-        , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
-    }
-}

trading/trader/.creds

@@ -1,12 +0,0 @@
-{
-    "__dummy__": null
-    , "ALPACA_SANDBOX": {
-        "api_key": "PKLZSLFZMFMN1R28K9HK"
-        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
-    }
-    , "MATTERMOST": {
-        "url": "https://mattermost.cryptovaltrading.com"
-        , "team": "CVTT"
-        , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
-    }
-}

trading/trading_recorder/.creds

@@ -1,12 +0,0 @@
-{
-    "__dummy__": null
-    , "ALPACA_SANDBOX": {
-        "api_key": "PKLZSLFZMFMN1R28K9HK"
-        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
-    }
-    , "MATTERMOST": {
-        "url": "https://mattermost.cryptovaltrading.com"
-        , "team": "CVTT"
-        , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
-    }
-}

trading/trading_recorder/run_trading_recorder.sh

@@ -14,7 +14,6 @@ usage() {
     exit 1
 }
 
-# ConfigServer=cloud16.cvtt.vpn
 ActiveExchanges=ALPACA_SNDBX-MDPORTAL
 AdminPort=7225
 Book=""