config (for executor) is made more flexible, docker compose initial

admin/docker_images.sh

@@ -0,0 +1,15 @@
+#!/bin/env bash
+
+Registry=http://homestore.cvtt.vpn:5500
+
+Catalog=${Registry}/v2/_catalog
+
+jstr=$(curl -s -X GET ${Catalog})
+
+# echo ${jstr}
+
+echo "${jstr}" | jq -r '.repositories[]' | while read repo; do
+    TagListURL="${Registry}/v2/${repo}/tags/list"
+    # echo $repo
+    curl -s -X GET ${TagListURL}
+done

doc/supervisord.conf.example

@@ -0,0 +1,61 @@
+[supervisord]
+environment=PYTHONPATH="/home/cvtt/prod"
+autostart=true
+autorestart=unexpected
+startretries=3
+logfile=/var/log/supervisor/supervisord_cvtt.log
+
+[supervisorctl]
+
+[inet_http_server]
+port = 127.0.0.1:9001
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[program:coinbase_rec_cvttdata]
+directory=/home/cvtt/prod
+command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
+        --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
+        --active_exchanges=COINBASE_AT
+        --instruments=COINBASE:PAIR-BTC-USD,COINBASE:PAIR-ETH-USD,COINBASE:PAIR-LTC-USD,COINBASE:PAIR-XRP-USD
+        --db_credentials_key=TSDB_MD_CVTTDATA
+        --admin_port=7223
+        --log_level=INFO
+        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.COINBASE.CVTTDATA.log
+autostart=true
+autorestart=true
+startretries=1
+startsecs=3
+user=cvtt
+#
+[program:bnbfut_rec_cvttdata]
+directory=/home/cvtt/prod
+command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
+        --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
+        --active_exchanges=BNBFUT
+        --instruments=BNBFUT:PERP-BTC-USDT,BNBFUT:PERP-ETH-USDT
+        --db_credentials_key=TSDB_MD_CVTTDATA
+        --admin_port=7224
+        --log_level=INFO
+        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.BNBFUT.CVTTDATA.log
+autostart=true
+autorestart=true
+startretries=1
+startsecs=3
+user=cvtt
+
+[program:md_rec_monitor]
+directory=/home/cvtt/prod
+command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_rec_monitor.py
+        --config=http://cloud16.cvtt.vpn:6789/apps/md_rec_monitor
+        --db=CVTTDATA
+        --admin_port=7225
+        --log_level=INFO
+        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.MONITOR.CVTTDATA.log
+autostart=true
+autorestart=true
+startretries=1
+startsecs=3
+user=cvtt
+

docker/trading/compose/.env

@@ -0,0 +1,5 @@
+BOOK=ALPACA-TEST-BOOK-01
+CONFIG=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
+ACTIVE_EXCHANGES=ALPACA_SNDBX-MDPORTAL
+#
+ADMIN_PORT_EXECUTOR=7222

docker/trading/compose/docker-compose.yml

@@ -0,0 +1,27 @@
+version: "3.8"
+#==================================================
+# Relies on the file `.env` content for varables:
+#    BOOK
+#    CONFIG
+#    ACTIVE_EXCHANGES
+#    ADMIN_PORT_EXECUTOR
+# Start with the command:
+#    docker compose up -d --pull
+#==================================================
+services:
+  redis:
+    image: redis/redis-stack:latest
+    container_name: redis-stack
+    ports:
+      - "6379:6379"
+  executor:
+    image: cloud21.cvtt.vpn:5500/executor:latest
+    container_name: executor.${BOOK}
+    network_mode: "host"
+    restart: unless-stopped
+    depends_on:
+      - redis
+    volumes:
+      - /opt/shared:/shared
+    command: ["-c", "${CONFIG}", "-e", "${ACTIVE_EXCHANGES}", "-a", "${ADMIN_PORT_EXECUTOR}"]
+    shm_size: "8gb"

docker/trading/executor/start_docker_executor.sh

@@ -2,7 +2,7 @@
 
 usage() {
   echo -n "Usage: $0"
-  echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
+  echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
   echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
   echo -n " [-a <admin_port (7222)>"]
   echo
@@ -11,7 +11,7 @@ usage() {
 
 args=${*}
 
-ConfigServer=cloud23.cvtt.vpn:6789
+Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
 ActiveExchanges=ALPACA_SNDBX-MDPORTAL
 AdminPort=7222
 Book=""
@@ -19,7 +19,7 @@ Book=""
 while getopts ":c:e:a:" opt; do
   case ${opt} in
     c )
-      ConfigServer=$OPTARG
+      Config=$OPTARG
       ;;
     e )
       ActiveExchanges=$OPTARG
@@ -38,7 +38,7 @@ while getopts ":c:e:a:" opt; do
   esac
 done
 
-echo "ConfigServer=${ConfigServer}"
+echo "Config=${Config}"
 echo "ActiveExchanges=${ActiveExchanges}"
 echo "AdminPort=${AdminPort}"
 

release_version.txt

@@ -1 +1 @@
-1.6.3,homestore /works/cvtt ownership moved to cvtt
+1.6.6,config (for executor) is made more flexible, docker compose initial

scripts/__DEPRECATED__/archive_yesterday_md.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3.10
+Python=/home/cvtt/.pyenv/python3.12-venv/bin/python3.12
 RootDir=/home/cvtt/prod
 export PYTHONPATH=${RootDir}
 

scripts/healthcheck/cvtt_ts_status_check.sh

@@ -10,7 +10,7 @@ usage() {
 }
 
 
-Python="${HOME}/.pyenv/python3.10-venv/bin/python3.10"
+Python="${HOME}/.pyenv/python3.12-venv/bin/python3.12"
 RootDir="${HOME}/prod"
 
 # **** D E B U G

scripts/mkt_data/alpaca_hist_md/alpaca_md.sh

@@ -45,7 +45,7 @@ echo "Start=${Start} End=${End} NumJobs=${NumJobs}"
 # export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_mkt_data.py
 
 export PYTHONPATH=/home/cvtt/prod
-export Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3
+export Python=/home/cvtt/.pyenv/python3.12-venv/bin/python3
 export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
 export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
 

scripts/mkt_data/alpaca_hist_md/load_rel_liq.sh

@@ -2,7 +2,7 @@
 
 export PYTHONPATH=${HOME}/prod
 
-Python=${HOME}/.pyenv/python3.10-venv/bin/python3
+Python=${HOME}/.pyenv/python3.12-venv/bin/python3
 Script=${HOME}/prod/cvttpy/exchanges/alpaca/hist_md/rl_calc_loader.py
 DbFile=${HOME}/prod/data/rel_liquidity.db
 Config=http://cloud23.cvtt.vpn:6789/apps/minimal_md_eqt

scripts/mkt_data/checklists/crypto_md_chcklst.sh

@@ -25,8 +25,8 @@ run_checklist() {
 
     declare -A Commands
     Commands=(
-        ["homestore:cloud21"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
-        ["homestore:cvttdata"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
+        ["homestore:cloud21"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
+        ["homestore:cvttdata"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
         ["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
         ["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
         ["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'"

scripts/mkt_data/checklists/eqty_md_chcklst.sh

@@ -24,9 +24,9 @@ run_checklist() {
     yr=$(date -d 'yesterday' '+%Y')
 
     declare -A Commands
-    Commands["homestore"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
+    Commands["homestore"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
     Commands["homestore"]+="; echo"
-    Commands["homestore"]+="; ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
+    Commands["homestore"]+="; ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
 
     Commands["cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
     Commands["cloud21"]+="; echo"
@@ -52,20 +52,19 @@ function cleanup {
 trap cleanup EXIT
 
 echo "## :fire: EQUITY MD HEALTH CHECK" >> ${tmpfile}
-echo  '```' >> ${tmpfile}
-run_checklist  >> ${tmpfile}
-echo  '```' >> ${tmpfile}
+echo  '```' | tee -a ${tmpfile}
+run_checklist  | tee -a ${tmpfile}
+echo  '```' | tee -a ${tmpfile}
 
+echo "Sending result to ${Sender} ${StatusChannel}"
 cat ${tmpfile} |  ${Sender} ${StatusChannel}
 
-
-
 exit
 # homestore
 echo homestore
-ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3
+ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3
 echo
-ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3
+ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3
 
 echo cloud21
 ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3

scripts/mkt_data/crypto_hist_md/exchanges_trading_stats.sh

@@ -2,7 +2,7 @@
 
 export PYTHONPATH=/home/cvtt/prod
 
-Cmd="/home/cvtt/.pyenv/python3.10-venv/bin/python3"
+Cmd="/home/cvtt/.pyenv/python3.12-venv/bin/python3"
 Cmd="${Cmd} /home/cvtt/prod/cvttpy/apps/research/exchange_trading_stats.py"
 Cmd="${Cmd} --config=http://cloud16.cvtt.vpn:6789/apps/tests/listen_market_data"
 Cmd="${Cmd} --active_exchanges=OKX,GEMINI,BITSTAMP,COINBASE_AT,BNBSPOT,KRAKEN"

scripts/mkt_data/legacy_eqt_md/process_md.sh

@@ -12,7 +12,7 @@ fi
 rsync -ahv /home/cvtt/host_drive/eqt_hist_md/txt/${letter} /home/cvtt/tmp/txt/
 
 # create db files
-PYTHONPATH=/home/cvtt/prod /home/cvtt/.pyenv/python3.10-venv/bin/python3 /home/cvtt/prod/cvttpy/trading/mkt_data/ad_hoc/eqt_md_to_db.py /home/cvtt/tmp/txt/${letter} /home/cvtt/tmp/db
+PYTHONPATH=/home/cvtt/prod /home/cvtt/.pyenv/python3.12-venv/bin/python3 /home/cvtt/prod/cvttpy/trading/mkt_data/ad_hoc/eqt_md_to_db.py /home/cvtt/tmp/txt/${letter} /home/cvtt/tmp/db
 
 if [ "$?" != "0" ] ; then
     exit

scripts/mkt_data/sync_market_data.sh

@@ -2,7 +2,7 @@
 
 Source=/home/cvtt/prod/archive/md_archive/
 Targets=
-Targets="${Targets} oleg@homestore.cvtt.vpn:/works/cvtt/md_archive/"
+Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/"
 Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/"