config (for all trading images) is made more flexible, docker compose initial

admin/docker_images.sh

@@ -0,0 +1,15 @@
+#!/bin/env bash
+
+Registry=http://homestore.cvtt.vpn:5500
+
+Catalog=${Registry}/v2/_catalog
+
+jstr=$(curl -s -X GET ${Catalog})
+
+# echo ${jstr}
+
+echo "${jstr}" | jq -r '.repositories[]' | while read repo; do
+    TagListURL="${Registry}/v2/${repo}/tags/list"
+    # echo $repo
+    curl -s -X GET ${TagListURL}
+done

doc/supervisord.conf.example

@@ -0,0 +1,61 @@
+[supervisord]
+environment=PYTHONPATH="/home/cvtt/prod"
+autostart=true
+autorestart=unexpected
+startretries=3
+logfile=/var/log/supervisor/supervisord_cvtt.log
+
+[supervisorctl]
+
+[inet_http_server]
+port = 127.0.0.1:9001
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[program:coinbase_rec_cvttdata]
+directory=/home/cvtt/prod
+command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
+        --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
+        --active_exchanges=COINBASE_AT
+        --instruments=COINBASE:PAIR-BTC-USD,COINBASE:PAIR-ETH-USD,COINBASE:PAIR-LTC-USD,COINBASE:PAIR-XRP-USD
+        --db_credentials_key=TSDB_MD_CVTTDATA
+        --admin_port=7223
+        --log_level=INFO
+        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.COINBASE.CVTTDATA.log
+autostart=true
+autorestart=true
+startretries=1
+startsecs=3
+user=cvtt
+#
+[program:bnbfut_rec_cvttdata]
+directory=/home/cvtt/prod
+command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
+        --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
+        --active_exchanges=BNBFUT
+        --instruments=BNBFUT:PERP-BTC-USDT,BNBFUT:PERP-ETH-USDT
+        --db_credentials_key=TSDB_MD_CVTTDATA
+        --admin_port=7224
+        --log_level=INFO
+        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.BNBFUT.CVTTDATA.log
+autostart=true
+autorestart=true
+startretries=1
+startsecs=3
+user=cvtt
+
+[program:md_rec_monitor]
+directory=/home/cvtt/prod
+command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_rec_monitor.py
+        --config=http://cloud16.cvtt.vpn:6789/apps/md_rec_monitor
+        --db=CVTTDATA
+        --admin_port=7225
+        --log_level=INFO
+        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.MONITOR.CVTTDATA.log
+autostart=true
+autorestart=true
+startretries=1
+startsecs=3
+user=cvtt
+

docker/market_data/md_portal/start_docker_md_portal.sh

@@ -2,8 +2,7 @@
 
 usage() {
     echo -n "Usage: ${0}"
-    echo -n " [-C <config (dflt: apps/cvtt_eqt_alpaca)>]"
+    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
-    echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
     echo -n " [-e <active_exchanges (ALPACA_SNBOX)>]"
     echo -n " [-a <admin_port (7220)>]"
     echo -n " [-n <portal_name (MD_PORTAL_ALPACA)>]"
@@ -14,19 +13,15 @@ usage() {
 
 args=${*}
 
-Config=apps/cvtt_eqt_alpaca
+Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
-ConfigServer=cloud23.cvtt.vpn:6789
 ActiveExchanges=ALPACA_SNDBX
 PortalName=MD_PORTAL_ALPACA_SNDBX
 AdminPort=7220
 
 while getopts ":C:c:e:a:n:" opt; do
     case ${opt} in
-        C )
-            Config=$OPTARG
-            ;;
         c )
-            ConfigServer=$OPTARG
+            Config=$OPTARG
             ;;
         e )
             ActiveExchanges=$OPTARG
@@ -49,7 +44,6 @@ while getopts ":C:c:e:a:n:" opt; do
 done
 
 echo "Config=${Config}"
-echo "ConfigServer=${ConfigServer}"
 echo "ActiveExchanges=${ActiveExchanges}"
 echo "PortalName=${PortalName}"
 echo "AdminPort=${AdminPort}"

docker/trading/compose/.env

@@ -0,0 +1,6 @@
+BOOK=ALPACA-TEST-BOOK-01
+CONFIG=/config/cvtt_eqty_alpaca.cfg
+ACTIVE_EXCHANGES=ALPACA_SNDBX-MDPORTAL
+#
+ADMIN_PORT_EXECUTOR=7222
+

docker/trading/compose/docker-compose.yml

@@ -0,0 +1,29 @@
+version: "3.8"
+#==================================================
+# Relies on the file `.env` content for varables:
+#    BOOK
+#    CONFIG
+#    ACTIVE_EXCHANGES
+#    ADMIN_PORT_EXECUTOR
+# Start with the command:
+#    docker compose up -d --pull=always
+#==================================================
+services:
+  redis:
+    image: redis/redis-stack:latest
+    container_name: redis-stack
+    ports:
+      - "6379:6379"
+  executor:
+    image: cloud21.cvtt.vpn:5500/executor:latest
+    container_name: executor.${BOOK}
+    network_mode: "host"
+    restart: unless-stopped
+    depends_on:
+      - redis
+    volumes:
+      - /opt/cvtt/shared:/shared
+      - /opt/cvtt/logs:/logs
+      - /opt/cvtt/config:/config
+    command: ["-c", "${CONFIG}", "-e", "${ACTIVE_EXCHANGES}", "-a", "${ADMIN_PORT_EXECUTOR}"]
+    shm_size: "8gb"

docker/trading/executor/start_docker_executor.sh

@@ -1,44 +1,44 @@
 #!/bin/bash
 
 usage() {
-  echo -n "Usage: $0"
+    echo -n "Usage: $0"
-  echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
+    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
-  echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+    echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
-  echo -n " [-a <admin_port (7222)>"]
+    echo -n " [-a <admin_port (7222)>"]
-  echo
+    echo
-  exit 1
+    exit 1
 }
 
 args=${*}
 
-ConfigServer=cloud23.cvtt.vpn:6789
+Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
 ActiveExchanges=ALPACA_SNDBX-MDPORTAL
 AdminPort=7222
 Book=""
 
 while getopts ":c:e:a:" opt; do
-  case ${opt} in
+    case ${opt} in
-    c )
+        c )
-      ConfigServer=$OPTARG
+            Config=$OPTARG
-      ;;
+            ;;
-    e )
+        e )
-      ActiveExchanges=$OPTARG
+            ActiveExchanges=$OPTARG
-      ;;
+            ;;
-    a )
+        a )
-      AdminPort=$OPTARG
+            AdminPort=$OPTARG
-      ;;
+            ;;
-    \? )
+        \? )
-      echo "Invalid option: -$OPTARG" >&2
+            echo "Invalid option: -$OPTARG" >&2
-      usage
+            usage
-      ;;
+            ;;
-    : )
+        : )
-      echo "Option -$OPTARG requires an argument." >&2
+            echo "Option -$OPTARG requires an argument." >&2
-      usage
+            usage
-      ;;
+            ;;
-  esac
+    esac
 done
 
-echo "ConfigServer=${ConfigServer}"
+echo "Config=${Config}"
 echo "ActiveExchanges=${ActiveExchanges}"
 echo "AdminPort=${AdminPort}"
 

docker/trading/quant/start_docker_quant.sh

@@ -1,21 +1,21 @@
 #!/bin/bash
 
 usage() {
-  echo -n "Usage: $0"
+    echo -n "Usage: $0"
-  echo -n " -b <book>"
+    echo -n " -b <book>"
 
-  echo -n " -S <strategy> (dflt: TRDALGO_001)"
+    echo -n " -S <strategy> (dflt: TRDALGO_001)"
 
-  echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
+    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
-  echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+    echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
-  echo -n " [-a <admin_port (7224)>"]
+    echo -n " [-a <admin_port (7224)>"]
-  echo
+    echo
-  exit 1
+    exit 1
 }
 
 args=${*}
 
-ConfigServer=cloud23.cvtt.vpn:6789
+Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
 ActiveExchanges=ALPACA_SNDBX-MDPORTAL
 AdminPort=7224
 Strategy=DAILY_STOCK_001
@@ -23,31 +23,31 @@ Book=""
 
 
 while getopts ":b:c:e:a:S:" opt; do
-  case ${opt} in
+    case ${opt} in
-    c )
+        c )
-      ConfigServer=$OPTARG
+            Config=$OPTARG
-      ;;
+            ;;
-    e )
+        e )
-      ActiveExchanges=$OPTARG
+            ActiveExchanges=$OPTARG
-      ;;
+            ;;
-    a )
+        a )
-      AdminPort=$OPTARG
+            AdminPort=$OPTARG
-      ;;
+            ;;
-    b )
+        b )
-      Book=$OPTARG
+            Book=$OPTARG
-      ;;
+            ;;
-    S )
+        S )
-      Strategy=$OPTARG
+            Strategy=$OPTARG
-      ;;
+            ;;
-    \? )
+        \? )
-      echo "Invalid option: -$OPTARG" >&2
+            echo "Invalid option: -$OPTARG" >&2
-      usage
+            usage
-      ;;
+            ;;
-    : )
+        : )
-      echo "Option -$OPTARG requires an argument." >&2
+            echo "Option -$OPTARG requires an argument." >&2
-      usage
+            usage
-      ;;
+            ;;
-  esac
+    esac
 done
 
 if [ "${Book}" == "" ]; then
@@ -55,7 +55,7 @@ if [ "${Book}" == "" ]; then
     usage
 fi
 
-echo "ConfigServer=${ConfigServer}"
+echo "Config=${Config}"
 echo "ActiveExchanges=${ActiveExchanges}"
 echo "Book=${Book}"
 echo "Strategy=${Strategy}"

docker/trading/risk_mgr/start_docker_risk_mgr.sh

@@ -1,51 +1,51 @@
 #!/bin/bash
 
 usage() {
-  echo -n "Usage: $0"
+    echo -n "Usage: $0"
-  echo -n " -b <book>"
+    echo -n " -b <book>"
-  echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
+    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
-  echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+    echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
-  echo -n " [-a <admin_port (7223)>"]
+    echo -n " [-a <admin_port (7223)>"]
-  echo
+    echo
-  exit 1
+    exit 1
 }
 
 
 args=${*}
 
-ConfigServer=cloud23.cvtt.vpn:6789
+Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
 ActiveExchanges=ALPACA_SNDBX-MDPORTAL
 AdminPort=7223
 Book=""
 
 
 while getopts ":b:c:e:a:" opt; do
-  case ${opt} in
+    case ${opt} in
-    c )
+        c )
-      ConfigServer=$OPTARG
+            Config=$OPTARG
-      ;;
+            ;;
-    e )
+        e )
-      ActiveExchanges=$OPTARG
+            ActiveExchanges=$OPTARG
-      ;;
+            ;;
-    a )
+        a )
-      AdminPort=$OPTARG
+            AdminPort=$OPTARG
-      ;;
+            ;;
-    b )
+        b )
-      Book=$OPTARG
+            Book=$OPTARG
-      ;;
+            ;;
-    \? )
+        \? )
-      echo "Invalid option: -$OPTARG" >&2
+            echo "Invalid option: -$OPTARG" >&2
-      usage
+            usage
-      ;;
+            ;;
-    : )
+        : )
-      echo "Option -$OPTARG requires an argument." >&2
+            echo "Option -$OPTARG requires an argument." >&2
-      usage
+            usage
-      ;;
+            ;;
-  esac
+    esac
 done
 
 
-echo "ConfigServer=${ConfigServer}"
+echo "Config=${Config}"
 echo "ActiveExchanges=${ActiveExchanges}"
 echo "Book=${Book}"
 echo "AdminPort=${AdminPort}"

docker/trading/trader/start_docker_trader.sh

@@ -1,23 +1,22 @@
 #!/bin/bash
 
 usage() {
-  echo -n "Usage: $0"
+    echo -n "Usage: $0"
-  echo -n " -b <book>"
+    echo -n " -b <book>"
 
-  echo -n " -A <algo> (dflt: TRDALGO_001)"
+    echo -n " -A <algo> (dflt: TRDALGO_001)"
 
-  echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
+    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
-  echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+    echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
-  echo -n " [-a <admin_port (7223)>"]
+    echo -n " [-a <admin_port (7223)>"]
-  echo
+    echo
-  exit 1
+    exit 1
 }
 
 
 args=${*}
 
-# ConfigServer=cloud16.cvtt.vpn
+Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
-ConfigServer=cloud23.cvtt.vpn:6789
 ActiveExchanges=ALPACA_SNDBX-MDPORTAL
 AdminPort=7226
 Algo=TRDALGO_001
@@ -25,31 +24,31 @@ Book=""
 
 
 while getopts ":b:c:e:a:" opt; do
-  case ${opt} in
+    case ${opt} in
-    c )
+        c )
-      ConfigServer=$OPTARG
+            Config=$OPTARG
-      ;;
+            ;;
-    e )
+        e )
-      ActiveExchanges=$OPTARG
+            ActiveExchanges=$OPTARG
-      ;;
+            ;;
-    a )
+        a )
-      AdminPort=$OPTARG
+            AdminPort=$OPTARG
-      ;;
+            ;;
-    b )
+        b )
-      Book=$OPTARG
+            Book=$OPTARG
-      ;;
+            ;;
-    A )
+        A )
-      Algo=$OPTARG
+            Algo=$OPTARG
-      ;;
+            ;;
-    \? )
+        \? )
-      echo "Invalid option: -$OPTARG" >&2
+            echo "Invalid option: -$OPTARG" >&2
-      usage
+            usage
-      ;;
+            ;;
-    : )
+        : )
-      echo "Option -$OPTARG requires an argument." >&2
+            echo "Option -$OPTARG requires an argument." >&2
-      usage
+            usage
-      ;;
+            ;;
-  esac
+    esac
 done
 
 if [ "${Book}" == "" ]; then
@@ -57,7 +56,7 @@ if [ "${Book}" == "" ]; then
     usage
 fi
 
-echo "ConfigServer=${ConfigServer}"
+echo "Config=${Config}"
 echo "ActiveExchanges=${ActiveExchanges}"
 echo "Book=${Book}"
 echo "Algo=${Algo}"

docker/trading/trading_recorder/start_docker_trading_recorder.sh

@@ -1,49 +1,46 @@
 #!/bin/bash
 
 usage() {
-  echo -n "Usage: $0"
+    echo -n "Usage: $0"
-  echo -n " -b <book>"
+    echo -n " -b <book>"
-  echo -n " [-C <config (dflt: apps/cvtt_eqt_alpaca)>]"
+    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
-  echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
+    echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
-  echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+    echo -n " [-a <admin_port (7223)>"]
-  echo -n " [-a <admin_port (7223)>"]
+    echo
-  echo
+    exit 1
-  exit 1
 }
 
 
 args=${*}
 
-# ConfigServer=cloud16.cvtt.vpn
+Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
-ConfigServer=cloud23.cvtt.vpn:6789
 ActiveExchanges=ALPACA_SNDBX-MDPORTAL
 AdminPort=7225
 Book=""
-Config=apps/cvtt_eqt_alpaca
 
 while getopts ":C:b:c:e:a:" opt; do
-  case ${opt} in
+    case ${opt} in
-    c )
+        c )
-      ConfigServer=$OPTARG
+            Config=$OPTARG
-      ;;
+            ;;
-    e )
+        e )
-      ActiveExchanges=$OPTARG
+            ActiveExchanges=$OPTARG
-      ;;
+            ;;
-    a )
+        a )
-      AdminPort=$OPTARG
+            AdminPort=$OPTARG
-      ;;
+            ;;
-    b )
+        b )
-      Book=$OPTARG
+            Book=$OPTARG
-      ;;
+            ;;
-    \? )
+        \? )
-      echo "Invalid option: -$OPTARG" >&2
+            echo "Invalid option: -$OPTARG" >&2
-      usage
+            usage
-      ;;
+            ;;
-    : )
+        : )
-      echo "Option -$OPTARG requires an argument." >&2
+            echo "Option -$OPTARG requires an argument." >&2
-      usage
+            usage
-      ;;
+            ;;
-  esac
+    esac
 done
 
 if [ "${Book}" == "" ]; then
@@ -52,7 +49,6 @@ if [ "${Book}" == "" ]; then
 fi
 
 echo "Config=${Config}"
-echo "ConfigServer=${ConfigServer}"
 echo "ActiveExchanges=${ActiveExchanges}"
 echo "Book=${Book}"
 echo "AdminPort=${AdminPort}"

release_version.txt

@@ -1 +1 @@
-1.6.3F2,homestore /works/cvtt ownership moved to cvtt
+1.6.7,config (for all trading images) is made more flexible, docker compose initial

scripts/__DEPRECATED__/archive_yesterday_md.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3.10
+Python=/home/cvtt/.pyenv/python3.12-venv/bin/python3.12
 RootDir=/home/cvtt/prod
 export PYTHONPATH=${RootDir}
 

scripts/healthcheck/cvtt_ts_status_check.sh

@@ -10,7 +10,7 @@ usage() {
 }
 
 
-Python="${HOME}/.pyenv/python3.10-venv/bin/python3.10"
+Python="${HOME}/.pyenv/python3.12-venv/bin/python3.12"
 RootDir="${HOME}/prod"
 
 # **** D E B U G

scripts/mkt_data/alpaca_hist_md/alpaca_md.sh

@@ -45,7 +45,7 @@ echo "Start=${Start} End=${End} NumJobs=${NumJobs}"
 # export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_mkt_data.py
 
 export PYTHONPATH=/home/cvtt/prod
-export Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3
+export Python=/home/cvtt/.pyenv/python3.12-venv/bin/python3
 export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
 export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
 

scripts/mkt_data/alpaca_hist_md/load_rel_liq.sh

@@ -2,7 +2,7 @@
 
 export PYTHONPATH=${HOME}/prod
 
-Python=${HOME}/.pyenv/python3.10-venv/bin/python3
+Python=${HOME}/.pyenv/python3.12-venv/bin/python3
 Script=${HOME}/prod/cvttpy/exchanges/alpaca/hist_md/rl_calc_loader.py
 DbFile=${HOME}/prod/data/rel_liquidity.db
 Config=http://cloud23.cvtt.vpn:6789/apps/minimal_md_eqt

scripts/mkt_data/checklists/eqty_md_chcklst.sh

@@ -52,10 +52,11 @@ function cleanup {
 trap cleanup EXIT
 
 echo "## :fire: EQUITY MD HEALTH CHECK" >> ${tmpfile}
-echo  '```' >> ${tmpfile}
+echo  '```' | tee -a ${tmpfile}
-run_checklist  >> ${tmpfile}
+run_checklist  | tee -a ${tmpfile}
-echo  '```' >> ${tmpfile}
+echo  '```' | tee -a ${tmpfile}
 
+echo "Sending result to ${Sender} ${StatusChannel}"
 cat ${tmpfile} |  ${Sender} ${StatusChannel}
 
 exit

scripts/mkt_data/crypto_hist_md/exchanges_trading_stats.sh

@@ -2,7 +2,7 @@
 
 export PYTHONPATH=/home/cvtt/prod
 
-Cmd="/home/cvtt/.pyenv/python3.10-venv/bin/python3"
+Cmd="/home/cvtt/.pyenv/python3.12-venv/bin/python3"
 Cmd="${Cmd} /home/cvtt/prod/cvttpy/apps/research/exchange_trading_stats.py"
 Cmd="${Cmd} --config=http://cloud16.cvtt.vpn:6789/apps/tests/listen_market_data"
 Cmd="${Cmd} --active_exchanges=OKX,GEMINI,BITSTAMP,COINBASE_AT,BNBSPOT,KRAKEN"

scripts/mkt_data/legacy_eqt_md/process_md.sh

@@ -12,7 +12,7 @@ fi
 rsync -ahv /home/cvtt/host_drive/eqt_hist_md/txt/${letter} /home/cvtt/tmp/txt/
 
 # create db files
-PYTHONPATH=/home/cvtt/prod /home/cvtt/.pyenv/python3.10-venv/bin/python3 /home/cvtt/prod/cvttpy/trading/mkt_data/ad_hoc/eqt_md_to_db.py /home/cvtt/tmp/txt/${letter} /home/cvtt/tmp/db
+PYTHONPATH=/home/cvtt/prod /home/cvtt/.pyenv/python3.12-venv/bin/python3 /home/cvtt/prod/cvttpy/trading/mkt_data/ad_hoc/eqt_md_to_db.py /home/cvtt/tmp/txt/${letter} /home/cvtt/tmp/db
 
 if [ "$?" != "0" ] ; then
     exit