20 changed files with 17 additions and 370 deletions
--- a/build.sh
+++ b/build.sh
@ -1,14 +1,14 @@
 #!/bin/bash

 function usage {
-    echo "Usage: ${0} <app_path (e.g. market_data/md_recorder_monitor)> <project>"
+    echo "Usage: ${0} <app_path (e.g. market_data/md_recorder_monitor)> <version>"
    exit
 }

 AppPath=${1}
-Project=${2}
+CvttpyVersion=${2}

-if [ "${Project}" == "" ]
+if [ "${AppPath}" == "" ]
 then
    usage
 fi
@ -24,9 +24,7 @@ ImageName=${AppName}
 ImageDir=${HOME}/docker_images
 mkdir -p ${ImageDir}

-
-Version=$(cat ${RootDir}/${Project}/release_version.txt | awk -F, '{print $1}')
-# Version=${ProjectVersion}
+Version=$(cat ${RootDir}/cvttpy/release_version.txt | awk -F, '{print $1}')

 cd ${RootDir}

--- a/market_data/alpaca_hbar/.creds
+++ b/market_data/alpaca_hbar/.creds
@ -1,7 +0,0 @@
-{
-    "__dummy__": null
-    , "ALPACA_SANDBOX": {
-        "api_key": "PKLZSLFZMFMN1R28K9HK"
-        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
-    }
-}
--- a/market_data/alpaca_hbar/Dockerfile
+++ b/market_data/alpaca_hbar/Dockerfile
@ -1,35 +0,0 @@
-FROM linuxcontainers/debian-slim:latest
-
-ARG FROM_DIR=./docker_dev/market_data/alpaca_hbar
-
-# Update the package list and install required packages
-RUN apt-get update && apt-get install -y \
-    --no-install-recommends \
-    apt-utils \
-    rsync \
-    openssh-client \
-    software-properties-common \
-    curl \
-    jq
-
-# must be from disted version
-COPY cvtt-rust/bin/alpaca_md_hbar_loader /
-
-COPY ${FROM_DIR}/.creds  /.creds
-COPY ${FROM_DIR}/alpaca_hbar.sh /alpaca_hbar.sh
-
-COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
-COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
-COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
-
-RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
-
-
-# Shared Volumes
-RUN mkdir -p /data
-RUN mkdir /logs
-RUN chmod +x /alpaca_hbar.sh
-
-WORKDIR /
-SHELL ["/bin/bash", "-c"]
-ENTRYPOINT [ "/alpaca_hbar.sh" ]
--- a/market_data/alpaca_hbar/alpaca_hbar.sh
+++ b/market_data/alpaca_hbar/alpaca_hbar.sh
@ -1,110 +0,0 @@
-#!/bin/bash
-
-usage() {
-  echo "Usage: $0 [-d <YYYYMMDD Date>] [-N <num_symbols>]"
-  exit 1
-}
-
-is_business_day() {
-    dt=${1}
-
-    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
-    if [ -n  "${open_time}" ]; then
-        return 0
-    else
-        return 1
-    fi
-
-}
-export -f is_business_day
-
-# --- Settings
-export Bin=/alpaca_md_hbar_loader
-export OutputDir=/data/alpaca_md # Local
-export LogDir=/logs
-# --- Settings
-
-while getopts ":d:N:" opt; do
-  case ${opt} in
-    d )
-      Start=$OPTARG
-      ;;
-    N )
-      NumSymbols=$OPTARG
-      ;;
-    \? )
-      echo "Invalid option: -$OPTARG" >&2
-      usage
-      ;;
-    : )
-      echo "Option -$OPTARG requires an argument." >&2
-      usage
-      ;;
-  esac
-done
-
-# if [ -z ${Symbols}  ] ; then
-#     echo "symbols are not specified"
-#     usage
-# fi
-
-mkdir -p ${OutputDir}
-mkdir -p ${LogDir}
-
-
-echo "CommandLine: ${*}"
-
-export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
-
-
-if [ -z ${Start}  ] ; then
-    echo "start is not specified, getting yesterday..."
-    Start=$(date -d "yesterday" "+%Y-%m-%d")
-    echo "Start=${Start}"
-fi
-
-while true; do
-    if is_business_day ${Start}; then
-        break
-    fi
-    echo "${Start} is not business day in US"
-    Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
-done
-
-
-echo "Start=${Start}"
-Dstamp=$(date -d "${Start}" "+%Y%m%d")
-
-Cmd="${Bin}"
-Cmd="${Cmd} --credentials_file=/.creds"
-Cmd="${Cmd} --output_directory=${OutputDir}"
-Cmd="${Cmd} --start=${Start}"
-if [ -n "${NumSymbols}" ]; then
-    Cmd="${Cmd} --num_symbols=${NumSymbols}"
-fi
-Cmd="${Cmd}  > ${LogDir}/${Dstamp}.alpaca_hbar.log 2>&1"
-
-echo ${Cmd}
-eval ${Cmd}
-
-echo "Compressing"
-for file in $(find ${OutputDir} -type f -name '*db' -print )
-do
-    echo "Compressing ${file}"
-    gzip ${file}
-done
-
-
-Source=/data/
-Targets=
-Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity"
-Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
-
-
-for tgt in ${Targets}
-do
-    Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
-    echo $Cmd
-    eval $Cmd
-done
-
--- a/market_data/_OLD_/alpaca_md_day/.creds
+++ b/market_data/_OLD_/alpaca_md_day/.creds
--- a/market_data/_OLD_/alpaca_md_day/.dockerignore
+++ b/market_data/_OLD_/alpaca_md_day/.dockerignore
--- a/market_data/_OLD_/alpaca_md_day/Dockerfile
+++ b/market_data/_OLD_/alpaca_md_day/Dockerfile
--- a/market_data/_OLD_/alpaca_md_day/HOWTO.md
+++ b/market_data/_OLD_/alpaca_md_day/HOWTO.md
--- a/market_data/_OLD_/alpaca_md_day/alpaca_md_day.sh
+++ b/market_data/_OLD_/alpaca_md_day/alpaca_md_day.sh
--- a/market_data/_OLD_/alpaca_md_day/requirements.txt
+++ b/market_data/_OLD_/alpaca_md_day/requirements.txt
--- a/market_data/alpaca_qat/.creds
+++ b/market_data/alpaca_qat/.creds
@ -1,7 +0,0 @@
-{
-    "__dummy__": null
-    , "ALPACA_SANDBOX": {
-        "api_key": "PKLZSLFZMFMN1R28K9HK"
-        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
-    }
-}
--- a/market_data/alpaca_qat/Dockerfile
+++ b/market_data/alpaca_qat/Dockerfile
@ -1,38 +0,0 @@
-FROM linuxcontainers/debian-slim:latest
-
-ARG FROM_DIR=docker_dev/market_data/alpaca_qat
-
-# Update the package list and install required packages
-RUN apt-get update && apt-get install -y \
-    --no-install-recommends \
-    apt-utils \
-    rsync \
-    openssh-client \
-    software-properties-common \
-    curl \
-    jq
-
-
-# must be from disted version
-COPY cvtt-rust/bin/alpaca_md_qat_loader /
-
-COPY ${FROM_DIR}/.creds  /.creds
-COPY ${FROM_DIR}/alpaca_qat.sh /alpaca_qat.sh
-
-COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
-COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
-COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
-
-RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
-
-
-# Shared Volumes
-RUN mkdir -p /data
-RUN mkdir /logs
-RUN chmod +x /alpaca_qat.sh
-
-WORKDIR /
-SHELL ["/bin/bash", "-c"]
-ENTRYPOINT [ "/alpaca_qat.sh" ]
-
-# CMD [ "echo", "alpaca_md_day"]
--- a/market_data/alpaca_qat/alpaca_qat.sh
+++ b/market_data/alpaca_qat/alpaca_qat.sh
@ -1,108 +0,0 @@
-#!/bin/bash
-
-usage() {
-  echo "Usage: $0 [-d <YYYYMMDD Date>] -S <symbols>"
-  exit 1
-}
-
-is_business_day() {
-    dt=${1}
-
-    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
-    if [ -n  "${open_time}" ]; then
-        return 0
-    else
-        return 1
-    fi
-
-}
-export -f is_business_day
-
-# --- Settings
-export Bin=/alpaca_md_qat_loader
-export OutputDir=/data/alpaca_md # Local
-export LogDir=/logs
-# --- Settings
-
-while getopts ":d:S:" opt; do
-  case ${opt} in
-    d )
-      Start=$OPTARG
-      ;;
-    S )
-      Symbols=$OPTARG
-      ;;
-    \? )
-      echo "Invalid option: -$OPTARG" >&2
-      usage
-      ;;
-    : )
-      echo "Option -$OPTARG requires an argument." >&2
-      usage
-      ;;
-  esac
-done
-
-if [ -z ${Symbols}  ] ; then
-    echo "symbols are not specified"
-    usage
-fi
-
-mkdir -p ${OutputDir}
-mkdir -p ${LogDir}
-
-
-echo "CommandLine: ${*}"
-
-export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
-
-
-if [ -z ${Start}  ] ; then
-    echo "start is not specified, getting yesterday..."
-    Start=$(date -d "yesterday" "+%Y-%m-%d")
-    echo "Start=${Start}"
-fi
-
-while true; do
-    if is_business_day ${Start}; then
-        break
-    fi
-    echo "${Start} is not business day in US"
-    Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
-done
-
-
-echo "Start=${Start}"
-Dstamp=$(date -d "${Start}" "+%Y%m%d")
-
-Cmd="${Bin}"
-Cmd="${Cmd} --credentials_file=/.creds"
-Cmd="${Cmd} --output_directory=${OutputDir}"
-Cmd="${Cmd} --start=${Start}"
-Cmd="${Cmd} --symbols=${Symbols}"
-Cmd="${Cmd}  > ${LogDir}/${Dstamp}.alpaca_qat.log 2>&1"
-
-echo ${Cmd}
-eval ${Cmd}
-
-echo "Compressing"
-for file in $(find ${OutputDir} -type f -name '*db' -print )
-do
-    echo "Compressing ${file}"
-    gzip ${file}
-done
-
-
-Source=/data/
-Targets=
-Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity"
-Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
-
-
-for tgt in ${Targets}
-do
-    Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
-    echo $Cmd
-    eval $Cmd
-done
-
--- a/market_data/md_recorder/run_md_recorder.sh
+++ b/market_data/md_recorder/run_md_recorder.sh
@ -34,32 +34,42 @@ Cmd="${Cmd} cvttpy/apps/md/md_recorder.py"
 Cmd="${Cmd} --config=http://${ConfigServer}/apps/md_recorder"
 Cmd="${Cmd} --credentials_file=/.creds"
 Cmd="${Cmd} --compress_log"
-Cmd="${Cmd} --instrument_group=${job}"
-Cmd="${Cmd} --log_file=/logs/%T.MD_REC.${job}.log"
 if [ "${job}" == "BNBFUT_CLD21" ] ; then
    Cmd="${Cmd} --active_exchanges=BNBFUT"
+    Cmd="${Cmd} --instruments=BNBFUT:PERP-BTC-USDT,BNBFUT:PERP-ETH-USDT"
    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
    Cmd="${Cmd} --admin_port=7201"
+    Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBFUT.CLD21.log"
 elif [ "${job}" == "COINBASE_CLD21" ] ; then
    Cmd="${Cmd} --active_exchanges=COINBASE_AT"
+    Cmd="${Cmd} --instruments=COINBASE:PAIR-BTC-USD,COINBASE:PAIR-ETH-USD,COINBASE:PAIR-LTC-USD,COINBASE:PAIR-XRP-USD"
    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
    Cmd="${Cmd} --admin_port=7202"
+    Cmd="${Cmd} --log_file=/logs/%T.MD_REC.COINBASE.CLD21.log"
 elif [ "${job}" == "BNBSPOT_CLD21" ] ; then
    Cmd="${Cmd} --active_exchanges=BNBSPOT"
+    Cmd="${Cmd} --instruments=BNBSPOT:PAIR-BTC-USDT,BNBSPOT:PAIR-ETH-USDT,BNBSPOT:PAIR-LTC-USDT,BNBSPOT:PAIR-XRP-USDT,BNBSPOT:PAIR-USDC-USDT"
    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
    Cmd="${Cmd} --admin_port=7203"
+    Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBSPOT.CLD21.log"
 elif [ "${job}" == "BNBSPOT_CVTTDATA" ] ; then
    Cmd="${Cmd} --active_exchanges=BNBSPOT"
+    Cmd="${Cmd} --instruments=BNBSPOT:PAIR-BTC-USDT,BNBSPOT:PAIR-ETH-USDT,BNBSPOT:PAIR-LTC-USDT,BNBSPOT:PAIR-XRP-USDT,BNBSPOT:PAIR-USDC-USDT"
    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
    Cmd="${Cmd} --admin_port=7204"
+    Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBSPOT.CVTTDATA.log"
 elif [ "${job}" == "BNBFUT_CVTTDATA" ] ; then
    Cmd="${Cmd} --active_exchanges=BNBFUT"
+    Cmd="${Cmd} --instruments=BNBFUT:PERP-BTC-USDT,BNBFUT:PERP-ETH-USDT"
    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
    Cmd="${Cmd} --admin_port=7205"
+    Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBFUT.CVTTDATA.log"
 elif [ "${job}" == "COINBASE_CVTTDATA" ] ; then
    Cmd="${Cmd} --active_exchanges=COINBASE_AT"
+    Cmd="${Cmd} --instruments=COINBASE:PAIR-BTC-USD,COINBASE:PAIR-ETH-USD,COINBASE:PAIR-LTC-USD,COINBASE:PAIR-XRP-USD"
    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
    Cmd="${Cmd} --admin_port=7206"
+    Cmd="${Cmd} --log_file=/logs/%T.MD_REC.COINBASE.CVTTDATA.log"
 else
    echo "Unrecognized JOB: ${job}"
    exit 1
--- a/market_data/relative_liquidity/Dockerfile
+++ b/market_data/relative_liquidity/Dockerfile
@ -17,7 +17,7 @@ CMD [                                        \
    "cvttpy/apps/md/rel_lqdt_server.py",    \
    "--port=5678",                           \
    "--db_file=/app/data/rel_liquidity.db",                      \
-    "--log_file=/logs/%T.REL_LIQUIDITY_SVC.log" \
+    "--log_file=/logs/%%T.REL_LIQUIDITY_SVC.log" \
 ]


--- a/release_version.txt
+++ b/release_version.txt
@ -1 +0,0 @@
-0.0.4,crypto_exch_stats
--- a/research/crypto_exch_stats/.dockerignore
+++ b/research/crypto_exch_stats/.dockerignore
@ -1,3 +0,0 @@
-.git
-**/__pycache__
-.pipenv
--- a/research/crypto_exch_stats/Dockerfile
+++ b/research/crypto_exch_stats/Dockerfile
@ -1,29 +0,0 @@
-FROM python:3.12-slim
-
-WORKDIR /
-COPY docker_dev/research/crypto_exch_stats/requirements.txt /
-
-RUN pip install --upgrade pip --root-user-action=ignore
-RUN pip install -r /requirements.txt --root-user-action=ignore
-
-COPY cvttpy /cvttpy
-
-# Shared Volumes
-RUN mkdir -p /app/data
-RUN mkdir /logs
-
-ENV PYTHONPATH=/
-CMD [                                                                      \
-    "python3.12",                                                          \
-    "cvttpy/apps//research/exchange_trading_stats.py",                     \
-    "--config=http://cloud23.cvtt.vpn:6789/apps/tests/listen_market_data", \
-    "--active_exchanges=OKX,GEMINI,BITSTAMP,COINBASE_AT,BNBSPOT,KRAKEN",   \
-    "--instruments=OKX:PAIR-BTC-USDT,GEMINI:PAIR-BTC-USD,BITSTAMP:PAIR-BTC-USD,COINBASE:PAIR-BTC-USD,BNBSPOT:PAIR-BTC-USDT,KRAKEN:PAIR-BTC-USD", \
-    "--db_file=/app/data/exchange_trading_stats.db",                       \
-    "--log_level=INFO",                                                    \
-    "--log_file=/logs/%T.EXCHANGE_TRADING_STATS.log"                       \
-]
-
-# ---------- No BNBSPOT from USA
-# "--active_exchanges=OKX,GEMINI,BITSTAMP,COINBASE_AT,KRAKEN",   \
-# "--instruments=OKX:PAIR-BTC-USDT,GEMINI:PAIR-BTC-USD,BITSTAMP:PAIR-BTC-USD,COINBASE:PAIR-BTC-USD,KRAKEN:PAIR-BTC-USD", \
--- a/research/crypto_exch_stats/requirements.txt
+++ b/research/crypto_exch_stats/requirements.txt
@ -1,7 +0,0 @@
-aiohttp>=3.7.4.post0
-nest-asyncio>=1.5.5
-numpy>=1.24.1
-hjson>=3.1.0
-pandas>=1.5.3
-sortedcontainers>=2.4.0
-redis>=5.0.8
--- a/research/crypto_exch_stats/run.sh
+++ b/research/crypto_exch_stats/run.sh
@ -1,16 +0,0 @@
-#!/bin/bash
-
-Cmd="docker run"
-Cmd+=" -d"
-# Cmd+=" --rm"
-Cmd+=" --pull=always"
-Cmd+=" --network=host"
-Cmd+=" --name=crypto_exch_stats"
-Cmd+=" --volume=${HOME}/prod/data:/app/data"
-Cmd+=" --volume=${HOME}/prod/logs:/logs"
-Cmd+=" cloud21.cvtt.vpn:5500/crypto_exch_stats:latest"
-
-echo ${Cmd}
-eval ${Cmd}
-# Cmd+=" cloud21.cvtt.vpn:5500/relative_liquidity:latest"
-