crypto_exch_stat

build.sh

@@ -1,14 +1,14 @@
 #!/bin/bash
 
 function usage {
-    echo "Usage: ${0} <app_path (e.g. market_data/md_recorder_monitor)> <version>"
+    echo "Usage: ${0} <app_path (e.g. market_data/md_recorder_monitor)> <project>"
     exit
 }
 
 AppPath=${1}
-CvttpyVersion=${2}
+Project=${2}
 
-if [ "${AppPath}" == "" ]
+if [ "${Project}" == "" ]
 then
     usage
 fi
@@ -24,7 +24,9 @@ ImageName=${AppName}
 ImageDir=${HOME}/docker_images
 mkdir -p ${ImageDir}
 
-Version=$(cat ${RootDir}/cvttpy/release_version.txt | awk -F, '{print $1}')
+
+Version=$(cat ${RootDir}/${Project}/release_version.txt | awk -F, '{print $1}')
+# Version=${ProjectVersion}
 
 cd ${RootDir}
 

market_data/alpaca_hbar/.creds

@@ -0,0 +1,7 @@
+{
+    "__dummy__": null
+    , "ALPACA_SANDBOX": {
+        "api_key": "PKLZSLFZMFMN1R28K9HK"
+        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
+    }
+}

market_data/alpaca_hbar/Dockerfile

@@ -0,0 +1,35 @@
+FROM linuxcontainers/debian-slim:latest
+
+ARG FROM_DIR=./docker_dev/market_data/alpaca_hbar
+
+# Update the package list and install required packages
+RUN apt-get update && apt-get install -y \
+    --no-install-recommends \
+    apt-utils \
+    rsync \
+    openssh-client \
+    software-properties-common \
+    curl \
+    jq
+
+# must be from disted version
+COPY cvtt-rust/bin/alpaca_md_hbar_loader /
+
+COPY ${FROM_DIR}/.creds  /.creds
+COPY ${FROM_DIR}/alpaca_hbar.sh /alpaca_hbar.sh
+
+COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
+COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
+COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
+
+RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
+
+
+# Shared Volumes
+RUN mkdir -p /data
+RUN mkdir /logs
+RUN chmod +x /alpaca_hbar.sh
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENTRYPOINT [ "/alpaca_hbar.sh" ]

market_data/alpaca_hbar/alpaca_hbar.sh

@@ -0,0 +1,110 @@
+#!/bin/bash
+
+usage() {
+  echo "Usage: $0 [-d <YYYYMMDD Date>] [-N <num_symbols>]"
+  exit 1
+}
+
+is_business_day() {
+    dt=${1}
+
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+# --- Settings
+export Bin=/alpaca_md_hbar_loader
+export OutputDir=/data/alpaca_md # Local
+export LogDir=/logs
+# --- Settings
+
+while getopts ":d:N:" opt; do
+  case ${opt} in
+    d )
+      Start=$OPTARG
+      ;;
+    N )
+      NumSymbols=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+# if [ -z ${Symbols}  ] ; then
+#     echo "symbols are not specified"
+#     usage
+# fi
+
+mkdir -p ${OutputDir}
+mkdir -p ${LogDir}
+
+
+echo "CommandLine: ${*}"
+
+export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+
+
+if [ -z ${Start}  ] ; then
+    echo "start is not specified, getting yesterday..."
+    Start=$(date -d "yesterday" "+%Y-%m-%d")
+    echo "Start=${Start}"
+fi
+
+while true; do
+    if is_business_day ${Start}; then
+        break
+    fi
+    echo "${Start} is not business day in US"
+    Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
+done
+
+
+echo "Start=${Start}"
+Dstamp=$(date -d "${Start}" "+%Y%m%d")
+
+Cmd="${Bin}"
+Cmd="${Cmd} --credentials_file=/.creds"
+Cmd="${Cmd} --output_directory=${OutputDir}"
+Cmd="${Cmd} --start=${Start}"
+if [ -n "${NumSymbols}" ]; then
+    Cmd="${Cmd} --num_symbols=${NumSymbols}"
+fi
+Cmd="${Cmd}  > ${LogDir}/${Dstamp}.alpaca_hbar.log 2>&1"
+
+echo ${Cmd}
+eval ${Cmd}
+
+echo "Compressing"
+for file in $(find ${OutputDir} -type f -name '*db' -print )
+do
+    echo "Compressing ${file}"
+    gzip ${file}
+done
+
+
+Source=/data/
+Targets=
+Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity"
+Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
+
+
+for tgt in ${Targets}
+do
+    Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
+    echo $Cmd
+    eval $Cmd
+done
+

market_data/alpaca_qat/.creds

@@ -0,0 +1,7 @@
+{
+    "__dummy__": null
+    , "ALPACA_SANDBOX": {
+        "api_key": "PKLZSLFZMFMN1R28K9HK"
+        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
+    }
+}

market_data/alpaca_qat/Dockerfile

@@ -0,0 +1,38 @@
+FROM linuxcontainers/debian-slim:latest
+
+ARG FROM_DIR=docker_dev/market_data/alpaca_qat
+
+# Update the package list and install required packages
+RUN apt-get update && apt-get install -y \
+    --no-install-recommends \
+    apt-utils \
+    rsync \
+    openssh-client \
+    software-properties-common \
+    curl \
+    jq
+
+
+# must be from disted version
+COPY cvtt-rust/bin/alpaca_md_qat_loader /
+
+COPY ${FROM_DIR}/.creds  /.creds
+COPY ${FROM_DIR}/alpaca_qat.sh /alpaca_qat.sh
+
+COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
+COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
+COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
+
+RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
+
+
+# Shared Volumes
+RUN mkdir -p /data
+RUN mkdir /logs
+RUN chmod +x /alpaca_qat.sh
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENTRYPOINT [ "/alpaca_qat.sh" ]
+
+# CMD [ "echo", "alpaca_md_day"]

market_data/alpaca_qat/alpaca_qat.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+
+usage() {
+  echo "Usage: $0 [-d <YYYYMMDD Date>] -S <symbols>"
+  exit 1
+}
+
+is_business_day() {
+    dt=${1}
+
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+# --- Settings
+export Bin=/alpaca_md_qat_loader
+export OutputDir=/data/alpaca_md # Local
+export LogDir=/logs
+# --- Settings
+
+while getopts ":d:S:" opt; do
+  case ${opt} in
+    d )
+      Start=$OPTARG
+      ;;
+    S )
+      Symbols=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+if [ -z ${Symbols}  ] ; then
+    echo "symbols are not specified"
+    usage
+fi
+
+mkdir -p ${OutputDir}
+mkdir -p ${LogDir}
+
+
+echo "CommandLine: ${*}"
+
+export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+
+
+if [ -z ${Start}  ] ; then
+    echo "start is not specified, getting yesterday..."
+    Start=$(date -d "yesterday" "+%Y-%m-%d")
+    echo "Start=${Start}"
+fi
+
+while true; do
+    if is_business_day ${Start}; then
+        break
+    fi
+    echo "${Start} is not business day in US"
+    Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
+done
+
+
+echo "Start=${Start}"
+Dstamp=$(date -d "${Start}" "+%Y%m%d")
+
+Cmd="${Bin}"
+Cmd="${Cmd} --credentials_file=/.creds"
+Cmd="${Cmd} --output_directory=${OutputDir}"
+Cmd="${Cmd} --start=${Start}"
+Cmd="${Cmd} --symbols=${Symbols}"
+Cmd="${Cmd}  > ${LogDir}/${Dstamp}.alpaca_qat.log 2>&1"
+
+echo ${Cmd}
+eval ${Cmd}
+
+echo "Compressing"
+for file in $(find ${OutputDir} -type f -name '*db' -print )
+do
+    echo "Compressing ${file}"
+    gzip ${file}
+done
+
+
+Source=/data/
+Targets=
+Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity"
+Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
+
+
+for tgt in ${Targets}
+do
+    Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
+    echo $Cmd
+    eval $Cmd
+done
+

market_data/md_recorder/run_md_recorder.sh

@@ -34,42 +34,32 @@ Cmd="${Cmd} cvttpy/apps/md/md_recorder.py"
 Cmd="${Cmd} --config=http://${ConfigServer}/apps/md_recorder"
 Cmd="${Cmd} --credentials_file=/.creds"
 Cmd="${Cmd} --compress_log"
+Cmd="${Cmd} --instrument_group=${job}"
+Cmd="${Cmd} --log_file=/logs/%T.MD_REC.${job}.log"
 if [ "${job}" == "BNBFUT_CLD21" ] ; then
     Cmd="${Cmd} --active_exchanges=BNBFUT"
-    Cmd="${Cmd} --instruments=BNBFUT:PERP-BTC-USDT,BNBFUT:PERP-ETH-USDT"
     Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
     Cmd="${Cmd} --admin_port=7201"
-    Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBFUT.CLD21.log"
 elif [ "${job}" == "COINBASE_CLD21" ] ; then
     Cmd="${Cmd} --active_exchanges=COINBASE_AT"
-    Cmd="${Cmd} --instruments=COINBASE:PAIR-BTC-USD,COINBASE:PAIR-ETH-USD,COINBASE:PAIR-LTC-USD,COINBASE:PAIR-XRP-USD"
     Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
     Cmd="${Cmd} --admin_port=7202"
-    Cmd="${Cmd} --log_file=/logs/%T.MD_REC.COINBASE.CLD21.log"
 elif [ "${job}" == "BNBSPOT_CLD21" ] ; then
     Cmd="${Cmd} --active_exchanges=BNBSPOT"
-    Cmd="${Cmd} --instruments=BNBSPOT:PAIR-BTC-USDT,BNBSPOT:PAIR-ETH-USDT,BNBSPOT:PAIR-LTC-USDT,BNBSPOT:PAIR-XRP-USDT,BNBSPOT:PAIR-USDC-USDT"
     Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
     Cmd="${Cmd} --admin_port=7203"
-    Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBSPOT.CLD21.log"
 elif [ "${job}" == "BNBSPOT_CVTTDATA" ] ; then
     Cmd="${Cmd} --active_exchanges=BNBSPOT"
-    Cmd="${Cmd} --instruments=BNBSPOT:PAIR-BTC-USDT,BNBSPOT:PAIR-ETH-USDT,BNBSPOT:PAIR-LTC-USDT,BNBSPOT:PAIR-XRP-USDT,BNBSPOT:PAIR-USDC-USDT"
     Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
     Cmd="${Cmd} --admin_port=7204"
-    Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBSPOT.CVTTDATA.log"
 elif [ "${job}" == "BNBFUT_CVTTDATA" ] ; then
     Cmd="${Cmd} --active_exchanges=BNBFUT"
-    Cmd="${Cmd} --instruments=BNBFUT:PERP-BTC-USDT,BNBFUT:PERP-ETH-USDT"
     Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
     Cmd="${Cmd} --admin_port=7205"
-    Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBFUT.CVTTDATA.log"
 elif [ "${job}" == "COINBASE_CVTTDATA" ] ; then
     Cmd="${Cmd} --active_exchanges=COINBASE_AT"
-    Cmd="${Cmd} --instruments=COINBASE:PAIR-BTC-USD,COINBASE:PAIR-ETH-USD,COINBASE:PAIR-LTC-USD,COINBASE:PAIR-XRP-USD"
     Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
     Cmd="${Cmd} --admin_port=7206"
-    Cmd="${Cmd} --log_file=/logs/%T.MD_REC.COINBASE.CVTTDATA.log"
 else
     echo "Unrecognized JOB: ${job}"
     exit 1

market_data/relative_liquidity/Dockerfile

@@ -17,7 +17,7 @@ CMD [                                        \
     "cvttpy/apps/md/rel_lqdt_server.py",    \
     "--port=5678",                           \
     "--db_file=/app/data/rel_liquidity.db",                      \
-    "--log_file=/logs/%%T.REL_LIQUIDITY_SVC.log" \
+    "--log_file=/logs/%T.REL_LIQUIDITY_SVC.log" \
 ]
 
 

release_version.txt

@@ -0,0 +1 @@
+0.0.4,crypto_exch_stats

research/crypto_exch_stats/.dockerignore

@@ -0,0 +1,3 @@
+.git
+**/__pycache__
+.pipenv

research/crypto_exch_stats/Dockerfile

@@ -0,0 +1,29 @@
+FROM python:3.12-slim
+
+WORKDIR /
+COPY docker_dev/research/crypto_exch_stats/requirements.txt /
+
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+
+# Shared Volumes
+RUN mkdir -p /app/data
+RUN mkdir /logs
+
+ENV PYTHONPATH=/
+CMD [                                                                      \
+    "python3.12",                                                          \
+    "cvttpy/apps//research/exchange_trading_stats.py",                     \
+    "--config=http://cloud23.cvtt.vpn:6789/apps/tests/listen_market_data", \
+    "--active_exchanges=OKX,GEMINI,BITSTAMP,COINBASE_AT,BNBSPOT,KRAKEN",   \
+    "--instruments=OKX:PAIR-BTC-USDT,GEMINI:PAIR-BTC-USD,BITSTAMP:PAIR-BTC-USD,COINBASE:PAIR-BTC-USD,BNBSPOT:PAIR-BTC-USDT,KRAKEN:PAIR-BTC-USD", \
+    "--db_file=/app/data/exchange_trading_stats.db",                       \
+    "--log_level=INFO",                                                    \
+    "--log_file=/logs/%T.EXCHANGE_TRADING_STATS.log"                       \
+]
+
+# ---------- No BNBSPOT from USA
+# "--active_exchanges=OKX,GEMINI,BITSTAMP,COINBASE_AT,KRAKEN",   \
+# "--instruments=OKX:PAIR-BTC-USDT,GEMINI:PAIR-BTC-USD,BITSTAMP:PAIR-BTC-USD,COINBASE:PAIR-BTC-USD,KRAKEN:PAIR-BTC-USD", \

research/crypto_exch_stats/requirements.txt

@@ -0,0 +1,7 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5
+numpy>=1.24.1
+hjson>=3.1.0
+pandas>=1.5.3
+sortedcontainers>=2.4.0
+redis>=5.0.8

research/crypto_exch_stats/run.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+Cmd="docker run"
+Cmd+=" -d"
+# Cmd+=" --rm"
+Cmd+=" --pull=always"
+Cmd+=" --network=host"
+Cmd+=" --name=crypto_exch_stats"
+Cmd+=" --volume=${HOME}/prod/data:/app/data"
+Cmd+=" --volume=${HOME}/prod/logs:/logs"
+Cmd+=" cloud21.cvtt.vpn:5500/crypto_exch_stats:latest"
+
+echo ${Cmd}
+eval ${Cmd}
+# Cmd+=" cloud21.cvtt.vpn:5500/relative_liquidity:latest"
+