rearrange + md monitor

build.sh

@@ -1,27 +1,28 @@
 #!/bin/bash
 
 function usage {
-    echo "Usage: ${0} <app_name>"
+    echo "Usage: ${0} <app_path (e.g. market_data/md_recorder_monitor)>"
     exit
 }
 
-AppName=${1}
+AppPath=${1}
 
-if [ "${AppName}" == "" ]
+if [ "${AppPath}" == "" ]
 then
     usage
 fi
 # --- Settings
 
-DockerDir=$(realpath $(dirname ${0})/${AppName})
-RootDir=$(realpath ${DockerDir}/../..)
+DockerDir=$(realpath $(dirname ${0})/${AppPath})
+RootDir=$(realpath $(dirname ${0})/..)
 RegistryService=cloud21.cvtt.vpn:5500
 
+AppName=$(basename ${AppPath})
+
 ImageName=${AppName}
 ImageDir=${HOME}/docker_images
 mkdir -p ${ImageDir}
 
-
 Version=$(cat ${RootDir}/cvttpy/release_version.txt)
 
 cd ${RootDir}

market_data/alpaca_md_day/Dockerfile

@@ -1,5 +1,7 @@
 FROM python:3.10-slim
 
+ARG FROM_DIR=docker_dev/market_data/alpaca_md_day
+
 # Update the package list and install required packages
 RUN apt-get update && apt-get install -y --no-install-recommends \
     apt-utils \
@@ -13,13 +15,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 RUN pip install --upgrade pip
 
 
-COPY docker_dev/alpaca_md_day/requirements.txt /
+COPY ${FROM_DIR}/requirements.txt /
 RUN pip install -r /requirements.txt
 
 COPY cvttpy /cvttpy
-COPY docker_dev/alpaca_md_day/.creds  /.creds
-COPY docker_dev/alpaca_md_day/.creds  /root/.creds
-COPY docker_dev/alpaca_md_day/alpaca_md_day.sh /alpaca_md_day.sh
+COPY ${FROM_DIR}/.creds  /.creds
+COPY ${FROM_DIR}/.creds  /root/.creds
+COPY ${FROM_DIR}/alpaca_md_day.sh /alpaca_md_day.sh
 
 COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
 COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub

market_data/crypto_md_day/Dockerfile

@@ -1,13 +1,14 @@
 FROM python:3.10-slim
 
+ARG FROM_DIR=docker_dev/market_data/crypto_md_day
 
-COPY docker_dev/crypto_md_day/requirements.txt /
+COPY ${FROM_DIR}/requirements.txt /
 RUN pip install --upgrade pip --root-user-action=ignore
 RUN pip install -r /requirements.txt --root-user-action=ignore
 
 COPY cvttpy /cvttpy
-COPY docker_dev/crypto_md_day/.creds  /.creds
-COPY docker_dev/crypto_md_day/crypto_md_day.sh /crypto_md_day.sh
+COPY ${FROM_DIR}/.creds  /.creds
+COPY ${FROM_DIR}/crypto_md_day.sh /crypto_md_day.sh
 
 RUN apt-get update && apt-get install -y --no-install-recommends rsync openssh-client && rm -rf /var/lib/apt/lists/*
 

market_data/md_recorder/Dockerfile

@@ -1,13 +1,15 @@
 FROM python:3.10-slim
 
+ARG FROM_DIR=docker_dev/market_data/md_recorder
 
-COPY docker_dev/md_recorder/requirements.txt /
+
+COPY ${FROM_DIR}/requirements.txt /
 RUN pip install --upgrade pip --root-user-action=ignore
 RUN pip install -r /requirements.txt --root-user-action=ignore
 
 COPY cvttpy /cvttpy
-COPY docker_dev/md_recorder/.creds  /.creds
-COPY docker_dev/md_recorder/run_md_recorder.sh /run_md_recorder.sh
+COPY ${FROM_DIR}/.creds  /.creds
+COPY ${FROM_DIR}/run_md_recorder.sh /run_md_recorder.sh
 
 
 # Shared Volumes

market_data/md_recorder/run_md_recorder.sh

@@ -2,24 +2,37 @@
 
 # runs in container
 
+ValidJobs=('BNBFUT_CLD21' 'BNBSPOT_CLD21' 'COINBASE_CLD21' 'BNBFUT_CVTTDATA' 'BNBSPOT_CVTTDATA' 'COINBASE_CVTTDATA')
+# runs on host to start container
 usage() {
-  echo "Usage: $0 <job_name> (BNBFUT_CLD21, BNBSPOT_CLD21, BNBSPOT_CVTTDATA, COINBASE_CLD21)"
+  echo "Usage: $0 <job_name; one of (${ValidJobs[@]})>"
   exit 1
 }
 
+is_valid() {
+  local job=$1
+  for valid_job in "${ValidJobs[@]}";
+  do
+    # echo "job=$job valid_job=$valid_job"
+    if [[ "${job}" == "${valid_job}" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
 
 job=${1}
-if [ "${job}" == "" ] ; then
+if  ! is_valid "${job}"; then
     usage
 fi
 
 # ConfigServer=cloud16.cvtt.vpn
-ConfigServer=cloud23.cvtt.vpn
+ConfigServer=cloud23.cvtt.vpn:6789
 
-if [ "${job}" == "BNBFUT_CLD21" ] ; then
 Cmd="python3.10"
 Cmd="${Cmd} cvttpy/apps/md/md_recorder.py"
-    Cmd="${Cmd} --config=http://${ConfigServer}:6789/apps/md_recorder"
+Cmd="${Cmd} --config=http://${ConfigServer}/apps/md_recorder"
+if [ "${job}" == "BNBFUT_CLD21" ] ; then
     Cmd="${Cmd} --active_exchanges=BNBFUT"
     Cmd="${Cmd} --instruments=BNBFUT:PERP-BTC-USDT,BNBFUT:PERP-ETH-USDT"
     Cmd="${Cmd} --credentials_file=/.creds"
@@ -27,9 +40,6 @@ if [ "${job}" == "BNBFUT_CLD21" ] ; then
     Cmd="${Cmd} --admin_port=7201"
     Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBFUT.CLD21.log"
 elif [ "${job}" == "COINBASE_CLD21" ] ; then
-    Cmd="python3.10"
-    Cmd="${Cmd} cvttpy/apps/md/md_recorder.py"
-    Cmd="${Cmd} --config=http://${ConfigServer}:6789/apps/md_recorder"
     Cmd="${Cmd} --active_exchanges=COINBASE_AT"
     Cmd="${Cmd} --instruments=COINBASE:PAIR-BTC-USD,COINBASE:PAIR-ETH-USD,COINBASE:PAIR-LTC-USD,COINBASE:PAIR-XRP-USD"
     Cmd="${Cmd} --credentials_file=/.creds"
@@ -37,9 +47,6 @@ elif [ "${job}" == "COINBASE_CLD21" ] ; then
     Cmd="${Cmd} --admin_port=7202"
     Cmd="${Cmd} --log_file=/logs/%T.MD_REC.COINBASE.CLD21.log"
 elif [ "${job}" == "BNBSPOT_CLD21" ] ; then
-    Cmd="python3.10"
-    Cmd="${Cmd} cvttpy/apps/md/md_recorder.py"
-    Cmd="${Cmd} --config=http://${ConfigServer}:6789/apps/md_recorder"
     Cmd="${Cmd} --active_exchanges=BNBSPOT"
     Cmd="${Cmd} --instruments=BNBSPOT:PAIR-BTC-USDT,BNBSPOT:PAIR-ETH-USDT,BNBSPOT:PAIR-LTC-USDT,BNBSPOT:PAIR-XRP-USDT,BNBSPOT:PAIR-USDC-USDT"
     Cmd="${Cmd} --credentials_file=/.creds"
@@ -47,15 +54,26 @@ elif [ "${job}" == "BNBSPOT_CLD21" ] ; then
     Cmd="${Cmd} --admin_port=7203"
     Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBSPOT.CLD21.log"
 elif [ "${job}" == "BNBSPOT_CVTTDATA" ] ; then
-    Cmd="python3.10"
-    Cmd="${Cmd} cvttpy/apps/md/md_recorder.py"
-    Cmd="${Cmd} --config=http://${ConfigServer}:6789/apps/md_recorder"
     Cmd="${Cmd} --active_exchanges=BNBSPOT"
     Cmd="${Cmd} --instruments=BNBSPOT:PAIR-BTC-USDT,BNBSPOT:PAIR-ETH-USDT,BNBSPOT:PAIR-LTC-USDT,BNBSPOT:PAIR-XRP-USDT,BNBSPOT:PAIR-USDC-USDT"
     Cmd="${Cmd} --credentials_file=/.creds"
     Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
     Cmd="${Cmd} --admin_port=7204"
     Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBSPOT.CVTTDATA.log"
+elif [ "${job}" == "BNBFUT_CVTTDATA" ] ; then
+    Cmd="${Cmd} --active_exchanges=BNBFUT"
+    Cmd="${Cmd} --instruments=BNBFUT:PERP-BTC-USDT,BNBFUT:PERP-ETH-USDT"
+    Cmd="${Cmd} --credentials_file=/.creds"
+    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
+    Cmd="${Cmd} --admin_port=7205"
+    Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBFUT.CVTTDATA.log"
+elif [ "${job}" == "COINBASE_CVTTDATA" ] ; then
+    Cmd="${Cmd} --active_exchanges=COINBASE_AT"
+    Cmd="${Cmd} --instruments=COINBASE:PAIR-BTC-USD,COINBASE:PAIR-ETH-USD,COINBASE:PAIR-LTC-USD,COINBASE:PAIR-XRP-USD"
+    Cmd="${Cmd} --credentials_file=/.creds"
+    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
+    Cmd="${Cmd} --admin_port=7206"
+    Cmd="${Cmd} --log_file=/logs/%T.MD_REC.COINBASE.CVTTDATA.log"
 else
     echo "Unrecognized JOB: ${job}"
     exit 1

market_data/md_recorder_monitor/.creds

@@ -0,0 +1,22 @@
+{
+    "__dummy__": null
+    , "MATTERMOST": {
+        "url": "https://mattermost.cryptovaltrading.com"
+        , "team": "CVTT"
+        , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
+    }
+    , "TSDB_MD_CLD21": {
+        "host": "cloud21.cvtt.vpn"
+        , "port": 5432
+        , "user": "cvtt"
+        , "database": "cvtt_md"
+        , "password": "ICdIh0JnMM7vM7Pf"
+    }
+    , "TSDB_MD_CVTTDATA": {
+        "host": "cvttdata.cvtt.vpn"
+        , "port": 5432
+        , "user": "cvtt"
+        , "database": "cvtt_md"
+        , "password": "ICdIh0JnMM7vM7Pf"
+    }
+}

market_data/md_recorder_monitor/Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.10-slim
+
+ARG FROM_DIR=docker_dev/market_data/md_recorder_monitor
+
+
+COPY ${FROM_DIR}/requirements.txt /
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+COPY ${FROM_DIR}/.creds  /.creds
+COPY ${FROM_DIR}/run_mdr_monitor.sh /run_mdr_monitor.sh
+
+
+# Shared Volumes
+RUN mkdir -p /app/data
+RUN mkdir /logs
+RUN chmod +x /run_mdr_monitor.sh
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENV PYTHONPATH=/
+ENTRYPOINT [ "/run_mdr_monitor.sh" ]
+

market_data/md_recorder_monitor/requirements.txt

@@ -0,0 +1,7 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5
+asyncpg>=0.27.0
+hjson>=3.1.0
+pandas>=1.5.3
+sortedcontainers>=2.4.0
+aioredis>=2.0.1

market_data/md_recorder_monitor/run_mdr_monitor.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# runs in container
+
+usage() {
+  echo "Usage: $0 <instance (CLD21,CVTTDATA)> [<admin_port (def. 7225)>]"
+  exit 1
+}
+
+ValidInstances=('CLD21' 'CVTTDATA')
+is_valid() {
+  local inst=$1
+  for valid_inst in "${ValidInstances[@]}";
+  do
+    if [[ "$inst" == "$valid_inst" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+
+Instance=${1}
+if ! is_valid "${Instance}"; then
+    usage
+fi
+
+AdminPort=7225
+if [ "${2}" != "" ]; then
+    AdminPort=${2}
+fi
+
+
+
+# ConfigServer=cloud16.cvtt.vpn
+ConfigServer=cloud23.cvtt.vpn:6789
+
+Cmd="python3.10"
+Cmd="${Cmd} cvttpy/apps/md/md_rec_monitor.py"
+Cmd="${Cmd} --config=http://c${ConfigServer}/apps/md_rec_monitor"
+Cmd="${Cmd} --db=${Instance}"
+Cmd="${Cmd} --admin_port=${AdminPort}"
+Cmd="${Cmd} --log_file=/logs/%%T.MD_REC.MONITOR.${Instance}.log
+
+
+echo ${Cmd}
+eval ${Cmd}
+
+