progress

docker/alpaca_md_day.sh

@@ -17,11 +17,42 @@ is_container_running() {
     fi
 }
 
+export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+
+is_business_day() {
+    dt=${1}
+
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+get_prev_business_day() {
+    Start=${1}
+    while true; do
+        if is_business_day ${Start}; then
+            break
+        fi
+        echo "${Start} is not business day in US" >&2
+        Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
+    done
+    echo ${Start}
+}
+export -f get_prev_business_day
 
 if [ -z ${date_to_load} ] ; then
     echo "Yesterday data has priority. Running historical container will be stopped"
     Cmd="docker stop ${ContainerName}"
     echo ${Cmd} && eval ${Cmd}
+    Cmd="docker kill ${ContainerName}"
+    echo ${Cmd} && eval ${Cmd}
+    Cmd="docker rm -f ${ContainerName}"
+    echo ${Cmd} && eval ${Cmd}
 
     Cmd="docker ps"
     echo ${Cmd} && eval ${Cmd}
@@ -64,6 +95,9 @@ else
         mv ${LastDayFile} ${LastDayFile}.prev
         date_to_load=$(date -d "${last_date} - 1 day" "+%Y-%m-%d")
     fi
+fi
+if [ -n "${date_to_load}" ]; then
+    date_to_load=$(get_prev_business_day ${date_to_load})
     echo "Historical Data for ${date_to_load}"
 fi
 

docker/config_service.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+Cmd="docker run"
+Cmd="${Cmd} -d"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --name=cvtt_config_service"
+Cmd="${Cmd} -p 6789:6789"
+Cmd="${Cmd} -v /home/cvtt/prod/config_service/data:/app/data"
+Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} cloud21.cvtt.vpn:5500/config_service:latest"
+
+echo ${Cmd}
+eval ${Cmd}

docker/md_recorder.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+ValidJobs=('BNBFUT_CLD21' 'BNBSPOT_CLD21' 'COINBASE_CLD21' 'BNBFUT_CVTTDATA' 'BNBSPOT_CVTTDATA' 'COINBASE_CVTTDATA')
+# runs on host to start container
+usage() {
+  echo "Usage: $0 <job_name; one of (${ValidJobs[@]})>"
+  exit 1
+}
+
+is_valid() {
+  local job=$1
+  for valid_job in "${ValidJobs[@]}";
+  do
+    # echo "job=$job valid_job=$valid_job"
+    if [[ "${job}" == "${valid_job}" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+is_container_running() {
+    local container_name=$1
+
+    if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
+        return 0  # true
+    else
+        return 1  # false
+    fi
+}
+
+
+job=${1}
+if  ! is_valid "${job}"; then
+    usage
+fi
+
+ContainerName="md_recorder.${job}"
+
+  if is_container_running "$ContainerName"; then
+      echo "Container ${ContainerName} is running."
+      exit
+  fi
+
+Cmd="docker run"
+Cmd="${Cmd} -d"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --name=${ContainerName}"
+Cmd="${Cmd} -v /home/cvtt/prod/data:/app/data"
+Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} cloud21.cvtt.vpn:5500/md_recorder:latest"
+Cmd="${Cmd} ${job}"
+
+echo ${Cmd}
+eval ${Cmd}

docker/md_recorder_monitor.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+usage() {
+  echo "Usage: $0 <instance (CLD21,CVTTDATA)> [<admin_port (def. 7225)>]"
+  exit 1
+}
+
+ValidInstances=('CLD21' 'CVTTDATA')
+is_valid() {
+  local inst=$1
+  for valid_inst in "${ValidInstances[@]}";
+  do
+    if [[ "$inst" == "$valid_inst" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+
+Instance=${1}
+if ! is_valid "${Instance}"; then
+    usage
+fi
+
+AdminPort=7225
+if [ "${2}" != "" ]; then
+    AdminPort=${2}
+fi
+
+
+Cmd="docker run"
+Cmd="${Cmd} -d"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --name=md_recorder_monitor.${Instance}"
+Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} cloud21.cvtt.vpn:5500/md_recorder_monitor:latest"
+Cmd="${Cmd} ${Instance} ${AdminPort}"
+
+echo ${Cmd}
+eval ${Cmd}

docker/relative_liquidity.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+Cmd="docker run"
+Cmd="${Cmd} -d"
+#Cmd="${Cmd} --rm"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --name=relative_liquidity_svc"
+Cmd="${Cmd} -p 5678:5678"
+Cmd="${Cmd} -v /home/cvtt/prod/data:/app/data"
+Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} cloud21.cvtt.vpn:5500/relative_liquidity:latest"
+
+echo ${Cmd}
+eval ${Cmd}

docker/trading_calendar.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+#Cmd="docker pull apptasticsoftware/trading-calendar:latest"
+#echo ${Cmd} && eval ${Cmd}
+
+Cmd="docker run"
+Cmd="${Cmd} -d"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --name trading-calendar"
+Cmd="${Cmd} -p 8000:80 apptasticsoftware/trading-calendar"
+echo ${Cmd} && eval ${Cmd}

release_version.txt

@@ -1 +1 @@
-1.0.9
+1.2.4

scripts/alpaca_hist_md/load_rel_liq.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+export PYTHONPATH=${HOME}/prod
+
+Python=${HOME}/.pyenv/python3.10-venv/bin/python3
+Script=${HOME}/prod/cvttpy/exchanges/alpaca/hist_md/rl_calc_loader.py
+DbFile=${HOME}/prod/data/rel_liquidity.db
+Config=http://cloud23.cvtt.vpn:6789/apps/minimal_md_eqt
+
+Cmd="${Python}"
+Cmd="${Cmd} ${Script}"
+Cmd="${Cmd} --config=${Config}"
+Cmd="${Cmd} --db_file=${DbFile}"
+
+echo ${Cmd}
+eval ${Cmd}
+
+echo Done ${0} ${*}

scripts/build_release.sh

@@ -7,25 +7,46 @@ git_repo_arr[cvttpy]=git@cloud21.cvtt.vpn:/opt/store/git/cvttpy.git
 git_repo_arr[ops]=git@cloud21.cvtt.vpn:/opt/store/git/ops.git
 git_repo_arr[research]=git@cloud21.cvtt.vpn:/opt/store/git/research.git
 
-default_project=cvttpy
-default_branch=master
-
-
 dist_root=/home/cvttdist/software/cvtt2
 dist_user=cvttdist
 dist_host="cloud21.cvtt.vpn"
 dist_ssh_port="22"
 
 dist_locations="cloud21.cvtt.vpn:22 homestore.cvtt.vpn:22"
+# ---------------- Settings
 
-interactive=Y
+# ---------------- cmdline
-
+prj=
-# cmdline
+brnch=master
-prj=${1:-${default_project}}
-brnch=${2:-${default_branch}}
-if [ "${3}" == "D" ]; then
 interactive=N
-fi
+
+usage() {
+  echo "Usage: $0 -p <project> [-b <branch (master)> -i (interactive)"
+  exit 1
+}
+
+while getopts ":p:b:i" opt; do
+  case ${opt} in
+    p )
+      prj=$OPTARG
+      ;;
+    b )
+      brnch=$OPTARG
+      ;;
+    i )
+      interactive=Y
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+# ---------------- cmdline
 
 function confirm {
     if [ "${interactive}" == "Y" ]; then

scripts/crypto_hist_md/exchanges_trading_stats.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+export PYTHONPATH=/home/cvtt/prod
+
+Cmd="/home/cvtt/.pyenv/python3.10-venv/bin/python3"
+Cmd="${Cmd} /home/cvtt/prod/cvttpy/apps/research/exchange_trading_stats.py"
+Cmd="${Cmd} --config=http://cloud16.cvtt.vpn:6789/apps/tests/listen_market_data"
+Cmd="${Cmd} --active_exchanges=OKX,GEMINI,BITSTAMP,COINBASE_AT,BNBSPOT,KRAKEN"
+Cmd="${Cmd} --instruments=OKX:PAIR-BTC-USDT,GEMINI:PAIR-BTC-USD,BITSTAMP:PAIR-BTC-USD,COINBASE:PAIR-BTC-USD,BNBSPOT:PAIR-BTC-USDT,KRAKEN:PAIR-BTC-USD"
+Cmd="${Cmd} --db_file=/home/cvtt/prod/data/exchange_trading_stats.db"
+Cmd="${Cmd} --log_file=/home/cvtt/prod/logs/%%T.EXCHANGE_TRADING_STATS.log"
+
+echo ${Cmd}
+eval ${Cmd}