config (for all trading images) is made more flexible, docker compose initial

admin/docker_images.sh

@@ -0,0 +1,15 @@
+#!/bin/env bash
+
+Registry=http://homestore.cvtt.vpn:5500
+
+Catalog=${Registry}/v2/_catalog
+
+jstr=$(curl -s -X GET ${Catalog})
+
+# echo ${jstr}
+
+echo "${jstr}" | jq -r '.repositories[]' | while read repo; do
+    TagListURL="${Registry}/v2/${repo}/tags/list"
+    # echo $repo
+    curl -s -X GET ${TagListURL}
+done

build/run_build.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+usage() {
+    echo "Usage: $0 <project name>"
+    exit 1
+}
+
+prj=${1}
+if [ "" == "${prj}" ] ; then
+    usage
+fi
+
+Cmd="pushd /home/oleg/develop/cvtt2"
+Cmd="${Cmd} && (cd ${prj}"
+Cmd="${Cmd} && git pushall)"
+Cmd="${Cmd} && ./build_release.sh -p ${prj}"
+Cmd="${Cmd} && (cd ${prj}"
+Cmd="${Cmd} && git pull --tags"
+Cmd="${Cmd} && git pushall)"
+Cmd="${Cmd} && popd"
+
+echo ${Cmd}
+eval ${Cmd}

doc/supervisord.conf.example

@@ -0,0 +1,61 @@
+[supervisord]
+environment=PYTHONPATH="/home/cvtt/prod"
+autostart=true
+autorestart=unexpected
+startretries=3
+logfile=/var/log/supervisor/supervisord_cvtt.log
+
+[supervisorctl]
+
+[inet_http_server]
+port = 127.0.0.1:9001
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[program:coinbase_rec_cvttdata]
+directory=/home/cvtt/prod
+command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
+        --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
+        --active_exchanges=COINBASE_AT
+        --instruments=COINBASE:PAIR-BTC-USD,COINBASE:PAIR-ETH-USD,COINBASE:PAIR-LTC-USD,COINBASE:PAIR-XRP-USD
+        --db_credentials_key=TSDB_MD_CVTTDATA
+        --admin_port=7223
+        --log_level=INFO
+        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.COINBASE.CVTTDATA.log
+autostart=true
+autorestart=true
+startretries=1
+startsecs=3
+user=cvtt
+#
+[program:bnbfut_rec_cvttdata]
+directory=/home/cvtt/prod
+command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
+        --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
+        --active_exchanges=BNBFUT
+        --instruments=BNBFUT:PERP-BTC-USDT,BNBFUT:PERP-ETH-USDT
+        --db_credentials_key=TSDB_MD_CVTTDATA
+        --admin_port=7224
+        --log_level=INFO
+        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.BNBFUT.CVTTDATA.log
+autostart=true
+autorestart=true
+startretries=1
+startsecs=3
+user=cvtt
+
+[program:md_rec_monitor]
+directory=/home/cvtt/prod
+command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_rec_monitor.py
+        --config=http://cloud16.cvtt.vpn:6789/apps/md_rec_monitor
+        --db=CVTTDATA
+        --admin_port=7225
+        --log_level=INFO
+        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.MONITOR.CVTTDATA.log
+autostart=true
+autorestart=true
+startretries=1
+startsecs=3
+user=cvtt
+

docker/market_data/hist/crypto_md_day.sh

@@ -14,7 +14,7 @@ Cmd="docker run"
 Cmd="${Cmd} --pull=always"
 Cmd="${Cmd} --network=host"
 Cmd="${Cmd} --name=crypto_md_day.${host}.${date}"
-# Cmd="${Cmd} --rm"
+Cmd="${Cmd} --rm"
 Cmd="${Cmd} ${DockerImage}"
 Cmd="${Cmd} -h ${host}"
 Cmd="${Cmd} -d ${date}"

docker/market_data/md_portal/start_docker_md_portal.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+usage() {
+    echo -n "Usage: ${0}"
+    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
+    echo -n " [-e <active_exchanges (ALPACA_SNBOX)>]"
+    echo -n " [-a <admin_port (7220)>]"
+    echo -n " [-n <portal_name (MD_PORTAL_ALPACA)>]"
+    echo
+    exit 1
+}
+
+
+args=${*}
+
+Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
+ActiveExchanges=ALPACA_SNDBX
+PortalName=MD_PORTAL_ALPACA_SNDBX
+AdminPort=7220
+
+while getopts ":C:c:e:a:n:" opt; do
+    case ${opt} in
+        c )
+            Config=$OPTARG
+            ;;
+        e )
+            ActiveExchanges=$OPTARG
+            ;;
+        a )
+            AdminPort=$OPTARG
+            ;;
+        n )
+            PortalName=$OPTARG
+            ;;
+        \? )
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            ;;
+        : )
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            ;;
+    esac
+done
+
+echo "Config=${Config}"
+echo "ActiveExchanges=${ActiveExchanges}"
+echo "PortalName=${PortalName}"
+echo "AdminPort=${AdminPort}"
+
+
+ImageName=md_portal
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/${ImageName}
+
+Cmd="docker run"
+Cmd="${Cmd} --detach"
+Cmd="${Cmd} --restart=unless-stopped"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ImageName}.${PortalName}"
+Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${args}"
+
+echo $Cmd
+eval $Cmd

docker/md_portal/start_docker_md_portal.sh

@@ -1,61 +0,0 @@
-#!/bin/bash
-
-args=${*}
-
-ConfigServer=cloud23.cvtt.vpn:6789
-ActiveExchanges=ALPACA_SNDBX
-PortalName=MD_PORTAL_ALPACA
-AdminPort=7220
-
-
-while getopts ":c:e:a:n:" opt; do
-  case ${opt} in
-    c )
-      ConfigServer=$OPTARG
-      ;;
-    e )
-      ActiveExchanges=$OPTARG
-      ;;
-    a )
-      AdminPort=$OPTARG
-      ;;
-    n )
-      PortalName=$OPTARG
-      ;;
-    \? )
-      echo "Invalid option: -$OPTARG" >&2
-      usage
-      ;;
-    : )
-      echo "Option -$OPTARG requires an argument." >&2
-      usage
-      ;;
-  esac
-done
-
-echo "ConfigServer=${ConfigServer}"
-echo "ActiveExchanges=${ActiveExchanges}"
-echo "PortalName=${PortalName}"
-echo "AdminPort=${AdminPort}"
-
-
-ImageName=md_portal
-DockerRegistry=cloud21.cvtt.vpn:5500
-DockerImage=${DockerRegistry}/${ImageName}
-
-if [ -z ${date} ] ; then
-    date=$(date -d "yesterday" +%Y%m%d)
-fi
-
-Cmd="docker run"
-# Cmd="${Cmd} --rm"
-Cmd="${Cmd} --restart=unless-stopped"
-Cmd="${Cmd} --pull=always"
-Cmd="${Cmd} --network=host"
-Cmd="${Cmd} --name=${ImageName}.${PortalName}"
-Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
-Cmd="${Cmd} ${DockerImage}"
-Cmd="${Cmd} ${args}"
-
-echo $Cmd
-eval $Cmd

docker/trading/compose/.env

@@ -0,0 +1,6 @@
+BOOK=ALPACA-TEST-BOOK-01
+CONFIG=/config/cvtt_eqty_alpaca.cfg
+ACTIVE_EXCHANGES=ALPACA_SNDBX-MDPORTAL
+#
+ADMIN_PORT_EXECUTOR=7222
+

docker/trading/compose/docker-compose.yml

@@ -0,0 +1,29 @@
+version: "3.8"
+#==================================================
+# Relies on the file `.env` content for varables:
+#    BOOK
+#    CONFIG
+#    ACTIVE_EXCHANGES
+#    ADMIN_PORT_EXECUTOR
+# Start with the command:
+#    docker compose up -d --pull=always
+#==================================================
+services:
+  redis:
+    image: redis/redis-stack:latest
+    container_name: redis-stack
+    ports:
+      - "6379:6379"
+  executor:
+    image: cloud21.cvtt.vpn:5500/executor:latest
+    container_name: executor.${BOOK}
+    network_mode: "host"
+    restart: unless-stopped
+    depends_on:
+      - redis
+    volumes:
+      - /opt/cvtt/shared:/shared
+      - /opt/cvtt/logs:/logs
+      - /opt/cvtt/config:/config
+    command: ["-c", "${CONFIG}", "-e", "${ACTIVE_EXCHANGES}", "-a", "${ADMIN_PORT_EXECUTOR}"]
+    shm_size: "8gb"

docker/trading/executor/start_docker_executor.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+usage() {
+    echo -n "Usage: $0"
+    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
+    echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+    echo -n " [-a <admin_port (7222)>"]
+    echo
+    exit 1
+}
+
+args=${*}
+
+Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
+ActiveExchanges=ALPACA_SNDBX-MDPORTAL
+AdminPort=7222
+Book=""
+
+while getopts ":c:e:a:" opt; do
+    case ${opt} in
+        c )
+            Config=$OPTARG
+            ;;
+        e )
+            ActiveExchanges=$OPTARG
+            ;;
+        a )
+            AdminPort=$OPTARG
+            ;;
+        \? )
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            ;;
+        : )
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            ;;
+    esac
+done
+
+echo "Config=${Config}"
+echo "ActiveExchanges=${ActiveExchanges}"
+echo "AdminPort=${AdminPort}"
+
+
+ImageName=executor
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/${ImageName}
+
+Cmd="docker run"
+Cmd="${Cmd} --detach"
+Cmd="${Cmd} --restart=unless-stopped"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ImageName}"
+Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${args}"
+
+echo $Cmd
+eval $Cmd

docker/trading/quant/start_docker_quant.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+usage() {
+    echo -n "Usage: $0"
+    echo -n " -b <book>"
+
+    echo -n " -S <strategy> (dflt: TRDALGO_001)"
+
+    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
+    echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+    echo -n " [-a <admin_port (7224)>"]
+    echo
+    exit 1
+}
+
+args=${*}
+
+Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
+ActiveExchanges=ALPACA_SNDBX-MDPORTAL
+AdminPort=7224
+Strategy=DAILY_STOCK_001
+Book=""
+
+
+while getopts ":b:c:e:a:S:" opt; do
+    case ${opt} in
+        c )
+            Config=$OPTARG
+            ;;
+        e )
+            ActiveExchanges=$OPTARG
+            ;;
+        a )
+            AdminPort=$OPTARG
+            ;;
+        b )
+            Book=$OPTARG
+            ;;
+        S )
+            Strategy=$OPTARG
+            ;;
+        \? )
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            ;;
+        : )
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            ;;
+    esac
+done
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+echo "Config=${Config}"
+echo "ActiveExchanges=${ActiveExchanges}"
+echo "Book=${Book}"
+echo "Strategy=${Strategy}"
+echo "AdminPort=${AdminPort}"
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+
+ImageName=quant
+
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/${ImageName}
+
+Cmd="docker run"
+Cmd="${Cmd} --detach"
+Cmd="${Cmd} --restart=unless-stopped"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ImageName}.${Book}"
+Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${args}"
+
+echo $Cmd
+eval $Cmd

docker/trading/risk_mgr/start_docker_risk_mgr.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+usage() {
+    echo -n "Usage: $0"
+    echo -n " -b <book>"
+    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
+    echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+    echo -n " [-a <admin_port (7223)>"]
+    echo
+    exit 1
+}
+
+
+args=${*}
+
+Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
+ActiveExchanges=ALPACA_SNDBX-MDPORTAL
+AdminPort=7223
+Book=""
+
+
+while getopts ":b:c:e:a:" opt; do
+    case ${opt} in
+        c )
+            Config=$OPTARG
+            ;;
+        e )
+            ActiveExchanges=$OPTARG
+            ;;
+        a )
+            AdminPort=$OPTARG
+            ;;
+        b )
+            Book=$OPTARG
+            ;;
+        \? )
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            ;;
+        : )
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            ;;
+    esac
+done
+
+
+echo "Config=${Config}"
+echo "ActiveExchanges=${ActiveExchanges}"
+echo "Book=${Book}"
+echo "AdminPort=${AdminPort}"
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+
+ImageName=risk_mgr
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/${ImageName}
+
+Cmd="docker run"
+Cmd="${Cmd} --detach"
+Cmd="${Cmd} --restart=unless-stopped"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ImageName}.${Book}"
+Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${args}"
+
+echo $Cmd
+eval $Cmd

docker/trading/trader/start_docker_trader.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+usage() {
+    echo -n "Usage: $0"
+    echo -n " -b <book>"
+
+    echo -n " -A <algo> (dflt: TRDALGO_001)"
+
+    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
+    echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+    echo -n " [-a <admin_port (7223)>"]
+    echo
+    exit 1
+}
+
+
+args=${*}
+
+Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
+ActiveExchanges=ALPACA_SNDBX-MDPORTAL
+AdminPort=7226
+Algo=TRDALGO_001
+Book=""
+
+
+while getopts ":b:c:e:a:" opt; do
+    case ${opt} in
+        c )
+            Config=$OPTARG
+            ;;
+        e )
+            ActiveExchanges=$OPTARG
+            ;;
+        a )
+            AdminPort=$OPTARG
+            ;;
+        b )
+            Book=$OPTARG
+            ;;
+        A )
+            Algo=$OPTARG
+            ;;
+        \? )
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            ;;
+        : )
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            ;;
+    esac
+done
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+echo "Config=${Config}"
+echo "ActiveExchanges=${ActiveExchanges}"
+echo "Book=${Book}"
+echo "Algo=${Algo}"
+echo "AdminPort=${AdminPort}"
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+
+ImageName=trader
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/${ImageName}
+
+Cmd="docker run"
+Cmd="${Cmd} --detach"
+Cmd="${Cmd} --restart=unless-stopped"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ImageName}.${Book}"
+Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${args}"
+
+echo $Cmd
+eval $Cmd

docker/trading/trading_recorder/start_docker_trading_recorder.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+usage() {
+    echo -n "Usage: $0"
+    echo -n " -b <book>"
+    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
+    echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+    echo -n " [-a <admin_port (7223)>"]
+    echo
+    exit 1
+}
+
+
+args=${*}
+
+Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
+ActiveExchanges=ALPACA_SNDBX-MDPORTAL
+AdminPort=7225
+Book=""
+
+while getopts ":C:b:c:e:a:" opt; do
+    case ${opt} in
+        c )
+            Config=$OPTARG
+            ;;
+        e )
+            ActiveExchanges=$OPTARG
+            ;;
+        a )
+            AdminPort=$OPTARG
+            ;;
+        b )
+            Book=$OPTARG
+            ;;
+        \? )
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            ;;
+        : )
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            ;;
+    esac
+done
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+echo "Config=${Config}"
+echo "ActiveExchanges=${ActiveExchanges}"
+echo "Book=${Book}"
+echo "AdminPort=${AdminPort}"
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+
+ImageName=trading_recorder
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/${ImageName}
+
+Cmd="docker run"
+Cmd="${Cmd} --detach"
+Cmd="${Cmd} --restart=unless-stopped"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ImageName}.${Book}"
+Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${args}"
+
+echo $Cmd
+eval $Cmd

release_version.txt

@@ -1 +1 @@
-1.4.4,start md_portal with command line validation
+1.6.7,config (for all trading images) is made more flexible, docker compose initial

research/load_crypto_md.sh

@@ -69,7 +69,7 @@ if [ "${host}" == "cloud21" ] ; then
     SourceRootDir="/opt/store/cvtt/md_archive/crypto/${source}"
 elif [ "${host}" == "homestore" ]; then
     SourceHost=homestore.cvtt.vpn
-    SourceUser=oleg
+    SourceUser=cvtt
     SourceRootDir=/works/cvtt/md_archive/crypto/${source}
 else
     usage
@@ -126,6 +126,5 @@ function download_file {
     chmod 600 ${TargetFilePath}
     ls -lh ${TargetFilePath}
 
-
 }
 download_file

research/load_eqty_md.sh

@@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+
+usage() {
+  echo -n "Usage: $0"
+  echo -n " [-h <host (homestore*/cloud21)>]"
+  echo -n " [-d <YYYYMMDD> (yesterday*)]"
+  echo -n " [-s <stocks comma separated>"
+  echo -n " [-t <target directory>"
+  echo
+  exit 1
+}
+
+if [[ "$(uname)" == "Darwin" ]]; then
+    # macOS
+    date='gdate'
+else
+    date='date'
+fi
+
+# ------------------ Settings
+md_date=""
+host=homestore
+stocks=COIN,GBTC,SQ
+
+TargetDir="/opt/jupyter_gpu/data/eqty_md"
+# TargetDir="/tmp"  # -------= D E B U G
+
+mkdir -p ${TargetDir}
+
+Table=md_1min_bars
+# ------------------ Settings
+
+
+
+is_valid() {
+  local inst=$1
+  shift
+  local valid_instances=("$@")
+
+  for valid_inst in "${valid_instances[@]}";
+  do
+    if [[ "$inst" == "$valid_inst" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+
+while getopts ":h:d:s:t:u" opt; do
+  case ${opt} in
+    d )
+      md_date=$OPTARG
+      ;;
+    h )
+      host=$OPTARG
+      ;;
+    s )
+      stocks=$OPTARG
+      ;;
+    t )
+      TargetDir=$OPTARG
+      ;;
+    u )
+      usage
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+if [ "${md_date}" == "" ] ; then
+    md_date="yesterday"
+fi
+
+echo "$md_date $host $source"
+
+valid_hosts=('homestore' 'cloud21')
+if ! is_valid "${host}"  "${valid_hosts[@]}" ; then
+    echo "Host '${host}' is not valid"
+    usage
+fi
+
+TargetFile=$(${date} -d ${md_date} "+%Y%m%d.eqty.mktdata.ohlcv.db")
+
+TargetFilePath="${TargetDir}/${TargetFile}"
+mv ${TargetFilePath} "${TargetFilePath}.saved.$(${date} '+%Y%m%d_%H%M%S')"
+
+# ---- temp dir
+echo ${SourceFile}
+tmp_dir=$(mktemp -d)
+
+function cleanup {
+    cd ${HOME}
+    rm -rf ${tmp_dir}
+}
+trap cleanup EXIT
+# ---- temp dir
+
+
+if [ "${host}" == "cloud21" ] ; then
+    SourceHost=cloud21.cvtt.vpn
+    SourceUser=cvtt
+    SourceRootDir="/opt/store/cvtt/md_archive/equity/alpaca_md"
+elif [ "${host}" == "homestore" ]; then
+    SourceHost=homestore.cvtt.vpn
+    SourceUser=cvtt
+    SourceRootDir="/works/cvtt/md_archive/equity/alpaca_md"
+else
+    usage
+fi
+
+
+OLDIFS=$IFS
+IFS=','
+read -ra Stocks <<< "$stocks"
+IFS=$OLDIFS
+
+
+echo "${SourceHost} ${SourceUser} ${SourceRootDir} ${Stocks[@]}"
+
+
+function download_file {
+    local SourceFilePath=${1}
+
+    Cmd="rsync"
+    Cmd="${Cmd} -ahv"
+    if tty -s; then
+        Cmd="${Cmd} --progress=info2"
+    fi
+    Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/"
+    echo ${Cmd}
+    eval ${Cmd}
+
+    if [[ ! -f ${tmp_dir}/${SourceFileZip} ]] ; then
+      echo "File ${SourceUser}@${SourceHost}:${SourceFilePath} is missing. Skipped."
+      return
+    fi
+
+    ls -lh ${tmp_dir}
+    Cmd="gunzip ${tmp_dir}/${SourceFileZip}"
+    echo ${Cmd} && eval ${Cmd}
+
+    DbFile=${tmp_dir}/${SourceFile}
+    touch ${TargetFilePath}
+
+
+# --- T E M P - for files older than 20240813
+    sqlite3 ${DbFile} <<EOF
+DROP INDEX md_1min_bars_idx;
+CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_idx on md_1min_bars(tstamp, exchange_id, instrument_id);
+EOF
+# --- T E M P
+
+    Cmd="sqlite3 ${DbFile} \".dump\" | sqlite3 ${TargetFilePath}"
+    echo ${Cmd}
+    eval ${Cmd}
+
+}
+
+echo "Loading files"
+for stock in "${Stocks[@]}"; do
+    StockLetter="${stock:0:1}"
+    SourceFile=$(${date} -d ${md_date} "+%Y%m%d.${stock}.1min.db")
+
+    SourceFileZip="${SourceFile}.gz"
+    SourceFilePath=$(${date} -d ${md_date} "+${SourceRootDir}/${StockLetter}/${stock}/%Y/${SourceFileZip}")
+    echo ${SourceFilePath}
+
+    download_file ${SourceFilePath}
+done
+
+if [ -f ${TargetFilePath} ]; then
+    chmod 600
+    ls -lh ${TargetFilePath}
+fi
+
+# mkdir -p /tmp/aaa
+# cp -r ${tmp_dir}/* /tmp/aaa/

scripts/__DEPRECATED__/archive_yesterday_md.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3.10
+Python=/home/cvtt/.pyenv/python3.12-venv/bin/python3.12
 RootDir=/home/cvtt/prod
 export PYTHONPATH=${RootDir}
 

scripts/__deprecated__/bnbspot_md_recorder_start.sh

@@ -1,156 +0,0 @@
-#!/bin/bash
-Script="${0} ${*}"
-
-function usage {
-    echo "Usage: ${0}  --inst_set=<setting #> [--restart] [--stop] [--once] [--sleep_sec=<num_seconds>]"
-    exit 1
-}
-
-for arg in "${@}"
-do
-    case ${arg} in
-        --restart)
-            Restart=Y
-            shift
-            ;;
-        --stop)
-            Stop="Y"
-            shift
-            ;;
-        --once)
-            Once="Y"
-            shift
-            ;;
-        --inst_set=*)
-            InstrSet="${arg#*=}"
-            shift
-            ;;
-        --sleep_sec=*)
-            SleepSec="${arg#*=}"
-            shift
-            ;;
-        -*|--*)
-            usage
-            ;;
-        *)
-            ;;
-    esac
-done
-
-
-declare -A InstrSetInstrs
-InstrSetInstrs[1]=PAIR-BTC-USDT,PAIR-ETH-USDT,PAIR-LTC-USDT,PAIR-XRP-USDT,PAIR-USDT-USD
-
-
-if [ "${InstrSet}" == "" ]
-then
-    usage
-fi
-
-if [ "${SleepSec}" == "" ]
-then
-    SleepSec=3
-fi
-
-Instruments=${InstrSetInstrs[${InstrSet}]}
-if [ "${Instruments}" == "" ]
-then
-    echo "Unrecognized instrument settings: ${InstrSet}"
-    usage
-fi
-
-# -------------------- S e t t i n g s
-RootDir=/home/cvtt/prod
-# ConfigFile="${RootDir}/config/md_recorder.cfg"
-ConfigFile="http://cloud11.cvtt.vpn:6789/apps/md_recorder"
-
-AdminPort="721${InstrSet}"
-Exchange=BNBSPOT
-DbCredKey=TSDB_COINBS_1
-Name=MD_RECORDER_BNBSPOT
-# -------------------- S e t t i n g s
-
-LogFile="${RootDir}/logs/$(date '+%Y%m%d_%H%M%S').${Name}_${InstrSet}.log"
-
-source ${HOME}/.pyenv/python3.10-venv/bin/activate
-export PYTHONPATH=${RootDir}
-
-Cmd="nohup"
-Cmd="${Cmd} python3"
-Cmd="${Cmd} ${RootDir}/cvttpy/apps/md/md_recorder.py"
-Cmd="${Cmd} --config=${ConfigFile}"
-Cmd="${Cmd} --active_exchanges=${Exchange}"
-Cmd="${Cmd} --instruments=${Instruments}"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_level=INFO"
-Cmd="${Cmd} --log_file=${LogFile}"
-Cmd="${Cmd} --db_credentials_key=${DbCredKey}"
-Cmd="${Cmd} &"
-
-function start_it {
-    echo ${Cmd}
-    eval ${Cmd} &
-    sleep 10
-}
-
-
-function check_it {
-
-    if !(timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    then
-        echo "${Script} doesn't respond. restarting..."
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-        start_it
-    fi
-}
-
-function kill_it {
-    pids=$(ps -ef |grep "inst_set=${InstrSet}" |grep $(basename ${0}) |grep -v ${$} |grep -v grep |tr -s ' ' |cut  -d' '  -f2)
-
-    if [ "${pids}" != "" ]
-    then
-        echo "Killing ${pids} ..."
-        kill -9 ${pids}
-    fi
-    while (timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    do
-        echo "Shutting down localhost:${AdminPort}/shutdown ..."
-        timeout 5 curl -s localhost:${AdminPort}/shutdown
-
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-    done
-}
-
-if [ "${Stop}" == "Y"  ] ; then
-    kill_it
-    exit 0
-fi
-
-if [ "${Once}" == "Y"  ] ; then
-    start_it
-    exit 0
-fi
-
-
-if [ "${Restart}" == "Y"  ]
-then
-    kill_it
-fi
-
-start_it
-while true
-do
-    check_it
-    echo "${Script} is checked" | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
-    sleep ${SleepSec}
-done

scripts/__deprecated__/coinbase_md_recorder_start.sh

@@ -1,161 +0,0 @@
-#!/bin/bash
-Script="${0} ${*}"
-
-function usage {
-    echo "Usage: ${0}  --inst_set=<setting #> [--restart] [--stop] [--once] [--sleep_sec=<num_seconds>]"
-    exit 1
-}
-
-for arg in "${@}"
-do
-    case ${arg} in
-        --restart)
-            Restart=Y
-            shift
-            ;;
-        --stop)
-            Stop="Y"
-            shift
-            ;;
-        --once)
-            Once="Y"
-            shift
-            ;;
-        --inst_set=*)
-            InstrSet="${arg#*=}"
-            shift
-            ;;
-        --sleep_sec=*)
-            SleepSec="${arg#*=}"
-            shift
-            ;;
-        -*|--*)
-            usage
-            ;;
-        *)
-            ;;
-    esac
-done
-
-
-declare -A InstrSetInstrs
-InstrSetInstrs[1]=PAIR-BTC-USD,PAIR-ETH-USD,PAIR-LTC-USD,PAIR-XRP-USD
-InstrSetInstrs[2]=PAIR-USDT-USD,PAIR-MATIC-USD
-InstrSetInstrs[3]=PAIR-DOGE-USD,PAIR-AVAX-USD,
-InstrSetInstrs[4]=PAIR-BCH-USD,PAIR-UNI-USD,PAIR-AAVE-USD,PAIR-LINK-USD,PAIR-SOL-USD
-
-
-if [ "${InstrSet}" == "" ]
-then
-    usage
-fi
-
-if [ "${SleepSec}" == "" ]
-then
-    SleepSec=3
-fi
-
-Instruments=${InstrSetInstrs[${InstrSet}]}
-if [ "${Instruments}" == "" ]
-then
-    echo "Unrecognized instrument settings: ${InstrSet}"
-    usage
-fi
-
-# -------------------- S e t t i n g s
-RootDir=/home/cvtt/prod
-# ConfigFile="${RootDir}/config/md_recorder.cfg"
-ConfigFile="http://cloud11.cvtt.vpn:6789/apps/md_recorder"
-
-AdminPort="720${InstrSet}"
-Exchange=COINBASE_AT
-DbCredKey=TSDB_COINBS_1
-Name=MD_RECORDER_COINBASE
-# -------------------- S e t t i n g s
-
-LogFile="${RootDir}/logs/$(date '+%Y%m%d_%H%M%S').${Name}_${InstrSet}.log"
-
-source ${HOME}/.pyenv/python3.10-venv/bin/activate
-export PYTHONPATH=${RootDir}
-
-Cmd="nohup"
-Cmd="${Cmd} python3"
-Cmd="${Cmd} ${RootDir}/cvttpy/apps/md/md_recorder.py"
-Cmd="${Cmd} --config=${ConfigFile}"
-Cmd="${Cmd} --active_exchanges=${Exchange}"
-Cmd="${Cmd} --instruments=${Instruments}"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_level=INFO"
-Cmd="${Cmd} --log_file=${LogFile}"
-Cmd="${Cmd} --db_credentials_key=${DbCredKey}"
-Cmd="${Cmd} &"
-
-function start_it {
-    echo ${Cmd}
-    eval ${Cmd} &
-    sleep 10
-}
-
-
-function check_it {
-
-    if !(timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    then
-        echo "${Script} doesn't respond. restarting..."
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-        start_it
-    fi
-}
-
-function kill_it {
-    pids=$(ps -ef |grep "inst_set=${InstrSet}" |grep $(basename ${0}) |grep -v ${$} |grep -v grep |tr -s ' ' |cut  -d' '  -f2)
-
-    if [ "${pids}" != "" ]
-    then
-        echo "Killing ${pids} ..."
-        kill -9 ${pids}
-    fi
-    while (timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    do
-        echo "Shutting down localhost:${AdminPort}/shutdown ..."
-        timeout 5 curl -s localhost:${AdminPort}/shutdown
-
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-    done
-}
-
-if [ "${Stop}" == "Y"  ] ; then
-    kill_it
-    exit 0
-fi
-
-if [ "${Once}" == "Y"  ] ; then
-    start_it
-    exit 0
-fi
-
-
-if [ "${Restart}" == "Y"  ]
-then
-    kill_it
-fi
-
-start_it
-while true
-do
-    check_it
-    echo "${Script} is checked" | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
-    sleep ${SleepSec}
-done
-
-

scripts/__deprecated__/config_server_start.sh

@@ -1,128 +0,0 @@
-#!/bin/bash
-Script="${0} ${*}"
-
-function usage {
-    echo "Usage: ${0}  [--restart] [--stop] [--once] [--sleep_sec=<num_seconds>]"
-    exit 1
-}
-
-for arg in "${@}"
-do
-    case ${arg} in
-        --restart)
-            Restart=Y
-            shift
-            ;;
-        --stop)
-            Stop="Y"
-            shift
-            ;;
-        --once)
-            Once="Y"
-            shift
-            ;;
-        --sleep_sec=*)
-            SleepSec="${arg#*=}"
-            shift
-            ;;
-        -*|--*)
-            usage
-            ;;
-        *)
-            ;;
-    esac
-done
-
-if [ "${SleepSec}" == "" ]
-then
-    SleepSec=5
-fi
-
-RootDir=/home/cvtt/prod
-CfgSvcDir=${RootDir}/config_service
-
-export PYTHONPATH="${RootDir}:${PYTHONPATH}"
-
-Name=config_server
-ServerPy="${RootDir}/cvttpy/apps/utils/config_server.py"
-ServicePort=6789
-ServiceDir=${CfgSvcDir}/data
-LogDir=${RootDir}/logs
-
-source ${HOME}/.pyenv/python3.10-venv/bin/activate
-export PYTHONPATH="${RootDir}:${PYTHONPATH}"
-
-Cmd="nohup"
-Cmd="${Cmd} python3"
-Cmd="${Cmd} ${ServerPy}"
-Cmd="${Cmd} --port=${ServicePort}"
-Cmd="${Cmd} --root=${ServiceDir}"
-Cmd="${Cmd} --log_file=${LogDir}/%T.config_service.log"
-
-function start_it {
-    echo ${Cmd}
-    eval ${Cmd} &
-    sleep 10
-}
-
-function check_it {
-    if !(timeout 3 curl -s localhost:${ServicePort}/ping > /dev/null)
-    then
-        echo "${Script} doesn't respond. restarting..."
-        pids=$(ps -ef | grep "port=${ServicePort}" | grep "root=${ServiceDir}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-        start_it
-    fi
-}
-
-function kill_it {
-    pids=$(ps -ef |grep $(basename ${0}) |grep -v ${$} |grep -v grep |tr -s ' ' |cut  -d' '  -f2)
-
-    if [ "${pids}" != "" ]
-    then
-        echo "Killing ${pids} ..."
-        kill -9 ${pids}
-    fi
-    while (timeout 3 curl -s localhost:${ServicePort}/ping > /dev/null)
-    do
-        echo "Shutting down localhost:${ServicePort}/__shutdown__ ..."
-        timeout 5 curl -s localhost:${ServicePort}/__shutdown__
-
-        pids=$(ps -ef | grep "port=${ServicePort}" | grep "root=${ServiceDir}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-    done
-}
-
-if [ "${Stop}" == "Y"  ] ; then
-    kill_it
-    exit 0
-fi
-
-if [ "${Once}" == "Y"  ] ; then
-    start_it
-    exit 0
-fi
-
-
-if [ "${Restart}" == "Y"  ]
-then
-    kill_it
-fi
-
-start_it
-while true
-do
-    check_it
-    echo "${Script} is checked" | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
-    sleep ${SleepSec}
-done
-
-

scripts/__deprecated__/executor_app_start.sh

@@ -1,144 +0,0 @@
-#!/bin/bash
-Script="${0} ${*}"
-
-function usage {
-    echo "Usage: ${0}  [--restart] [--stop] [--once] [--sleep_sec=<num_seconds>]"
-    exit 1
-}
-
-for arg in "${@}"
-do
-    case ${arg} in
-        --restart)
-            Restart=Y
-            shift
-            ;;
-        --stop)
-            Stop="Y"
-            shift
-            ;;
-        --once)
-            Once="Y"
-            shift
-            ;;
-        --inst_set=*)
-            InstrSet="${arg#*=}"
-            shift
-            ;;
-        --sleep_sec=*)
-            SleepSec="${arg#*=}"
-            shift
-            ;;
-        -*|--*)
-            usage
-            ;;
-        *)
-            ;;
-    esac
-done
-
-
-if [ "${SleepSec}" == "" ]
-then
-    SleepSec=3
-fi
-
-
-# -------------------- S e t t i n g s
-ConfigServiceHost=cloud11.cvtt.vpn
-ConfigServicePort=6789
-
-RootDir=/home/cvtt/prod
-
-ConfigFile="http://${ConfigServiceHost}:${ConfigServicePort}/apps/executor_app"
-
-AdminPort="7210"
-Exchange=COINBASE_AT
-Name=EXECUTOR_APP
-# -------------------- S e t t i n g s
-
-LogFile="${RootDir}/logs/$(date '+%Y%m%d_%H%M%S').${Name}.log"
-
-source ${HOME}/.pyenv/python3.10-venv/bin/activate
-export PYTHONPATH=${RootDir}
-
-Cmd="nohup"
-Cmd="${Cmd} python3"
-Cmd="${Cmd} ${RootDir}/cvttpy/apps/executor_app.py"
-Cmd="${Cmd} --config=${ConfigFile}"
-Cmd="${Cmd} --active_exchanges=${Exchange}"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_level=INFO"
-Cmd="${Cmd} --log_file=${LogFile}"
-Cmd="${Cmd} &"
-
-function start_it {
-    echo ${Cmd}
-    eval ${Cmd} &
-    sleep 10
-}
-
-
-function check_it {
-
-    if !(timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    then
-        echo "${Script} doesn't respond. restarting..."
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-        start_it
-    fi
-}
-
-function kill_it {
-    pids=$(ps -ef |grep $(basename ${0}) |grep -v ${$} |grep -v grep |tr -s ' ' |cut  -d' '  -f2)
-
-    if [ "${pids}" != "" ]
-    then
-        echo "Killing ${pids} ..."
-        kill -9 ${pids}
-    fi
-    while (timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    do
-        echo "Shutting down localhost:${AdminPort}/shutdown ..."
-        timeout 5 curl -s localhost:${AdminPort}/shutdown
-        sleep 3
-
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-    done
-}
-
-if [ "${Stop}" == "Y"  ] ; then
-    kill_it
-    exit 0
-fi
-
-if [ "${Once}" == "Y"  ] ; then
-    start_it
-    exit 0
-fi
-
-
-if [ "${Restart}" == "Y"  ]
-then
-    kill_it
-fi
-
-start_it
-while true
-do
-    check_it
-    echo "${Script} is checked" | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
-    sleep ${SleepSec}
-done
-
-

scripts/healthcheck/cvtt_ts_status_check.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+usage() {
+    echo -n "Usage: ${0}"
+    echo -n " [-c <config (dflt: apps/cvtt_eqt_alpaca)>]"
+    echo -n " [-s <config_serverice (dflt: http://cloud23.cvtt.vpn:6789)>]"
+    echo -n " [-N <name (dflt: ALPACA-SNDBX)>]"
+    echo
+    exit 1
+}
+
+
+Python="${HOME}/.pyenv/python3.12-venv/bin/python3.12"
+RootDir="${HOME}/prod"
+
+# **** D E B U G
+RootDir=/home/oleg/develop/cvtt2
+# **** D E B U G
+
+
+export PYTHONPATH=${RootDir}
+
+StatusChannel=Status-CVTT
+AlertChannel=Alerts-CVTT
+Sender=${RootDir}/ops/utils/send_mmost.sh
+ConfigService=http://cloud23.cvtt.vpn:6789
+Config=apps/cvtt_eqt_alpaca
+Name="ALPACA-SNDBX"
+
+echo $0 $* | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
+
+while getopts ":s:c:n:" opt; do
+    case ${opt} in
+        c )
+            Config=$OPTARG
+            ;;
+        s )
+            ConfigService=$OPTARG
+            ;;
+        n )
+            Name=$OPTARG
+            ;;
+        \? )
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            ;;
+        : )
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            ;;
+    esac
+done
+
+# ----- For DEBUGGING
+# Sender=cat
+# StatusChannel=
+
+tmpfile=$(mktemp)
+function cleanup {
+    # echo Cleaing up temporary files: ${TempFiles}
+    if [ "" != "${TempFiles}" ]; then
+        rm -f ${TempFiles}
+    fi
+}
+trap cleanup EXIT
+
+
+Cmd="${Python}"
+Cmd="${Cmd} ${RootDir}/cvttpy/apps/utils/services_checker.py"
+Cmd="${Cmd} --config=${ConfigService}/${Config}"
+Cmd="${Cmd} --log_level=ERROR"
+Cmd="${Cmd} --log_stdout"
+Cmd="${Cmd} | tee ${tmpfile} 2>&1"
+
+echo ${Cmd}
+eval ${Cmd}
+
+cat ${tmpfile}
+exit
+
+
+# echo "## :card_file_box: STORAGE HEALTH CHECK" >> ${tmpfile}
+# echo  >> ${tmpfile}
+# echo "| host | filesystem | usage |"  >> ${tmpfile}
+# echo "| --- | --- | --- |"  >> ${tmpfile}
+# cat ${tmpfile2} | sort -h -r | awk -F'%' '{printf "%s%%%s\n",$2,$3}' >> ${tmpfile}
+# cat ${tmpfile} |  ${Sender} ${StatusChannel}
+

scripts/mkt_data/alpaca_hist_md/alpaca_md.sh

@@ -45,7 +45,7 @@ echo "Start=${Start} End=${End} NumJobs=${NumJobs}"
 # export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_mkt_data.py
 
 export PYTHONPATH=/home/cvtt/prod
-export Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3
+export Python=/home/cvtt/.pyenv/python3.12-venv/bin/python3
 export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
 export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
 

scripts/mkt_data/alpaca_hist_md/load_rel_liq.sh

@@ -2,7 +2,7 @@
 
 export PYTHONPATH=${HOME}/prod
 
-Python=${HOME}/.pyenv/python3.10-venv/bin/python3
+Python=${HOME}/.pyenv/python3.12-venv/bin/python3
 Script=${HOME}/prod/cvttpy/exchanges/alpaca/hist_md/rl_calc_loader.py
 DbFile=${HOME}/prod/data/rel_liquidity.db
 Config=http://cloud23.cvtt.vpn:6789/apps/minimal_md_eqt

scripts/mkt_data/checklists/crypto_md_chcklst.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+usage() {
+    echo -n "Usage: ${0}"
+    echo
+    exit 1
+}
+
+
+RootDir="${HOME}/prod"
+# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
+
+export PYTHONPATH=${RootDir}
+
+StatusChannel="MD-Status"
+Sender=${RootDir}/ops/utils/send_mmost.sh
+
+# ----- For DEBUGGING
+# Sender=cat
+# StatusChannel=
+
+run_checklist() {
+    yr=$(date -d 'yesterday' '+%Y')
+    mn=$(date -d 'yesterday' '+%m')
+
+    declare -A Commands
+    Commands=(
+        ["homestore:cloud21"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
+        ["homestore:cvttdata"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
+        ["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
+        ["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
+        ["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'"
+    )
+
+    for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
+        Cmd=${Commands[${name}]}
+        echo "------- ${name}"
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+}
+
+tmpfile=$(mktemp)
+function cleanup {
+    cd ${HOME}
+    rm -f ${tmpfile}
+}
+trap cleanup EXIT
+
+echo "## :hearts: CRYPTO MD HEALTH CHECK" >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+run_checklist  >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+
+cat ${tmpfile} |  ${Sender} ${StatusChannel}
+
+cat $tmpfile
+

scripts/mkt_data/checklists/eqty_md_chcklst.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+usage() {
+    echo -n "Usage: ${0}"
+    echo
+    exit 1
+}
+
+RootDir="${HOME}/prod"
+# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
+
+export PYTHONPATH=${RootDir}
+
+StatusChannel="MD-Status"
+Sender=${RootDir}/ops/utils/send_mmost.sh
+
+# ----- For DEBUGGING
+# Sender=cat
+# StatusChannel=
+
+run_checklist() {
+
+
+    yr=$(date -d 'yesterday' '+%Y')
+
+    declare -A Commands
+    Commands["homestore"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
+    Commands["homestore"]+="; echo"
+    Commands["homestore"]+="; ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
+
+    Commands["cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
+    Commands["cloud21"]+="; echo"
+    Commands["cloud21"]+="; ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
+
+    Commands["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'"
+
+    for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
+        Cmd=${Commands[${name}]}
+        echo "------- ${name}"
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+}
+
+# run_checklist
+
+tmpfile=$(mktemp)
+function cleanup {
+    cd ${HOME}
+    rm -f ${tmpfile}
+}
+trap cleanup EXIT
+
+echo "## :fire: EQUITY MD HEALTH CHECK" >> ${tmpfile}
+echo  '```' | tee -a ${tmpfile}
+run_checklist  | tee -a ${tmpfile}
+echo  '```' | tee -a ${tmpfile}
+
+echo "Sending result to ${Sender} ${StatusChannel}"
+cat ${tmpfile} |  ${Sender} ${StatusChannel}
+
+exit
+# homestore
+echo homestore
+ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3
+echo
+ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3
+
+echo cloud21
+ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3
+echo
+ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3
+
+echo "------ gpushnik"
+ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'

scripts/mkt_data/crypto_hist_md/exchanges_trading_stats.sh

@@ -2,7 +2,7 @@
 
 export PYTHONPATH=/home/cvtt/prod
 
-Cmd="/home/cvtt/.pyenv/python3.10-venv/bin/python3"
+Cmd="/home/cvtt/.pyenv/python3.12-venv/bin/python3"
 Cmd="${Cmd} /home/cvtt/prod/cvttpy/apps/research/exchange_trading_stats.py"
 Cmd="${Cmd} --config=http://cloud16.cvtt.vpn:6789/apps/tests/listen_market_data"
 Cmd="${Cmd} --active_exchanges=OKX,GEMINI,BITSTAMP,COINBASE_AT,BNBSPOT,KRAKEN"

scripts/mkt_data/legacy_eqt_md/process_md.sh

@@ -12,7 +12,7 @@ fi
 rsync -ahv /home/cvtt/host_drive/eqt_hist_md/txt/${letter} /home/cvtt/tmp/txt/
 
 # create db files
-PYTHONPATH=/home/cvtt/prod /home/cvtt/.pyenv/python3.10-venv/bin/python3 /home/cvtt/prod/cvttpy/trading/mkt_data/ad_hoc/eqt_md_to_db.py /home/cvtt/tmp/txt/${letter} /home/cvtt/tmp/db
+PYTHONPATH=/home/cvtt/prod /home/cvtt/.pyenv/python3.12-venv/bin/python3 /home/cvtt/prod/cvttpy/trading/mkt_data/ad_hoc/eqt_md_to_db.py /home/cvtt/tmp/txt/${letter} /home/cvtt/tmp/db
 
 if [ "$?" != "0" ] ; then
     exit

scripts/mkt_data/sync_market_data.sh

@@ -2,7 +2,7 @@
 
 Source=/home/cvtt/prod/archive/md_archive/
 Targets=
-Targets="${Targets} oleg@homestore.cvtt.vpn:/works/cvtt/md_archive/"
+Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/"
 Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/"
 
 

utils/git/create_remote_repo.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# ---------------------
+# user "git" owned remote repository
+#
+# must run on remote server
+# change GitTopDir, HostName and RemoteName
+# ---------------------
+# R e s u l t  E x a m p l e:
+#     sudo bash -c 'mkdir /opt/store/git/cvtt2/tests.git && cd /opt/store/git/cvtt2/tests.git && git init --bare && chown -R git:git /opt/store/git/cvtt2/tests.git'
+# ---------------------
+#
+
+# ----- S e t t i n g s
+GitTopDir=/ops/store/git
+HostName=cloud21.cvtt.vpn
+RemoteName=origin
+# ----- S e t t i n g s
+
+usage() {
+    echo "Usage: ${0} <repo_name> [<repo_parent_dir> *${GitTopDir}/cvtt2]"
+    exit 1
+}
+
+
+RepoName=${1}
+if [ "" == "${RepoName}" ] ; then
+    usage
+fi
+
+RepoParentDir=${GitTopDir}/cvtt2
+if [ "${2}" != "" ] ; then
+    RepoParentDir=${2}
+fi
+
+RepoDir="${RepoParentDir}/${RepoName}.git"
+
+
+Cmd="sudo bash -c 'mkdir ${RepoDir}"
+Cmd="${Cmd} && cd ${RepoDir}"
+Cmd="${Cmd} && git init --bare "
+Cmd="${Cmd} && chown -R git:git ${RepoDir}'"
+echo ${Cmd}
+eval ${Cmd} || exit 1
+
+echo "====================="
+echo "Repository is created. Use this command to add it:"
+echo "     git remote add ${RemoteName} git@${HostName}:${RepoDir}"
+echo "====================="