progress

cronjobs/cvtt/cloud16.cron

@@ -1,10 +0,0 @@
-0 6 * * * /usr/bin/supervisorctl -c /home/cvtt/prod/config/supervisor/supervisord.conf restart config_service 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).config_service.restart.log
-#
-#
-#---------------------- Utils
-0 5 * * * /home/cvtt/prod/utils/config_svc_backup.sh 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).config_svc_backup.log
-0 6 * * * /home/cvtt/prod/utils/archive_logs.sh /home/cvtt/prod/logs 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).archive_logs.log
-10 1 * * * /home/cvtt/prod/utils/move_archives.sh  2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).move_archives.log
-
-#
-#---------------------- T E S T

cronjobs/cvtt/cloud17.cron

@@ -1,11 +0,0 @@
-0 6 * * * /usr/bin/supervisorctl -c /home/cvtt/prod/config/supervisor/supervisord.conf restart all 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).bnbspot_md_recorder.restart.log
-#
-#10 6 * * * /usr/bin/supervisorctl -c /home/cvtt/prod/config/supervisor/supervisord.conf restart executor 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).executor_app.restart.log
-#
-#---------------------- Utils
-0 6 * * * /home/cvtt/prod/utils/archive_logs.sh /home/cvtt/prod/logs 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).cronjob_log_archive.log
-10 1 * * * /home/cvtt/prod/utils/move_archives.sh cloud17 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).move_archives.log
-#
-# ----------------- #  TEST
-#
-# ----------------- #  TEST

cronjobs/cvtt/cloudstore.cron

@@ -1,8 +0,0 @@
-20 5 * * SAT /home/cvtt/prod/run/load_histdata.sh 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /home/cvtt/prod/logs/$(date +\
-20 6 * * SAT /home/cvtt/prod/run/prune_eqt_histdata.sh 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /home/cvtt/prod/logs/$(da
-#
-0 6 * * SUN /home/cvtt/prod/run/archive_logs.sh /home/cvtt/prod/logs 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/p
-#
-# T E S T
-#----------------------------------------------------------------------
-#----------------------------------------------------------------------

cronjobs/cvtt/cryptoval3.cron

@@ -1,11 +0,0 @@
-#
-#---------------------- Market Data
-0 6 * * * /home/cvtt/prod/run/archive_yesterday_md.sh  coinbase,bnbspot,bnbfut 2>&1  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).archive_yestd_md.log
-#---------------------- Utils
-0 7 * * * /home/cvtt/prod/utils/backup_md_archive.sh  2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).backup_md_archive.log
-#
-30 7 * * * /home/cvtt/prod/utils/archive_logs.sh /home/cvtt/prod/logs 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).cronjob_log_archive.log
-#
-# ----------------- #  TEST
-#
-# ----------------- #  TEST

cronjobs/save_cronjobs.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+
+usage() {
+    echo "Usage: ${0} <local_root_dir>"
+    exit 1
+}
+
+echo "Started ${0} ${*} ..."
+
+Domain=cvtt.vpn
+RootDir=${1}
+
+if [ ! -d "${RootDir}" ]; then
+    usage
+fi
+cd ${RootDir}
+
+for User in cvtt oleg
+do
+    echo "User=$User"
+    # hosts=$(curl -s http://cloud23.cvtt.vpn:6789/admin/cvtt_hosts | jq -r --arg domain "${Domain}" --arg usr ${User} '.[$domain] | to_entries[] | select(.value.users[] | contains($usr)) | .key')
+    hosts=$(
+        curl -s http://cloud23.cvtt.vpn:6789/admin/cvtt_hosts | jq -r \
+            --arg domain "${Domain}" \
+            --arg usr "${User}" \
+            '.[$domain] | to_entries[] | select(.value.users[] | contains($usr)) | .key'
+    )
+    mkdir -p ${RootDir}/${User}
+    for host in ${hosts}
+    do
+        echo "===================="
+        echo "${host}:${User}"
+        Cmd="ssh ${User}@${host}.${Domain} 'crontab -l' | tee ${RootDir}/${User}/${host}.cron"
+        echo ${Cmd} && eval ${Cmd}
+    done
+done
+
+cd ${RootDir}
+
+Cmd="git add ."
+echo ${Cmd} && eval ${Cmd}
+
+Cmd="git commit -a -m '$(date)'"
+echo ${Cmd} && eval ${Cmd}
+
+Cmd="git pushall"
+echo ${Cmd} && eval ${Cmd}
+
+echo "Done ${0} ${*}"

docker/alpaca_md_day.sh

@@ -100,7 +100,6 @@ if [ -n "${date_to_load}" ]; then
     date_to_load=$(get_prev_business_day ${date_to_load})
     echo "Historical Data for ${date_to_load}"
 fi
-exit
 
 Cmd="docker run"
 Cmd="${Cmd} --pull=always"

docker/md_recorder.sh

@@ -1,27 +1,58 @@
 #!/bin/bash
 
+ValidJobs=('BNBFUT_CLD21' 'BNBSPOT_CLD21' 'COINBASE_CLD21' 'BNBFUT_CVTTDATA' 'BNBSPOT_CVTTDATA' 'COINBASE_CVTTDATA')
 # runs on host to start container
 usage() {
-  echo "Usage: $0 <job_name> (BNBFUT_CLD21, BNBSPOT_CLD21, BNBSPOT_CVTTDATA, COINBASE_CLD21)"
+  echo "Usage: $0 <job_name; one of (${ValidJobs[@]})>"
   exit 1
 }
 
+is_valid() {
+  local job=$1
+  for valid_job in "${ValidJobs[@]}";
+  do
+    # echo "job=$job valid_job=$valid_job"
+    if [[ "${job}" == "${valid_job}" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+is_container_running() {
+    local container_name=$1
+
+    if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
+        return 0  # true
+    else
+        return 1  # false
+    fi
+}
+
 
 job=${1}
-if [ "${job}" == "" ] ; then
+if  ! is_valid "${job}"; then
     usage
 fi
 
+ContainerName="md_recorder.${job}"
+
+if is_container_running "$ContainerName"; then
+    echo "Container ${ContainerName} is already running. Aborted."
+    exit
+fi
+
+
 Cmd="docker run"
 Cmd="${Cmd} -d"
 Cmd="${Cmd} --rm"
 Cmd="${Cmd} --network=host"
 Cmd="${Cmd} --pull=always"
-Cmd="${Cmd} --name=md_recorder.${job}"
+Cmd="${Cmd} --name=${ContainerName}"
 Cmd="${Cmd} -v /home/cvtt/prod/data:/app/data"
 Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
 Cmd="${Cmd} cloud21.cvtt.vpn:5500/md_recorder:latest"
-Cmd="${Cmd} ${1}"
+Cmd="${Cmd} ${job}"
 
 echo ${Cmd}
 eval ${Cmd}

docker/md_recorder_monitor.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+usage() {
+  echo "Usage: $0 <instance (CLD21,CVTTDATA)> [<admin_port (def. 7225)>]"
+  exit 1
+}
+
+ValidInstances=('CLD21' 'CVTTDATA')
+is_valid() {
+  local inst=$1
+  for valid_inst in "${ValidInstances[@]}";
+  do
+    if [[ "$inst" == "$valid_inst" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+is_container_running() {
+    local container_name=$1
+
+    if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
+        return 0  # true
+    else
+        return 1  # false
+    fi
+}
+
+
+
+
+Instance=${1}
+if ! is_valid "${Instance}"; then
+    usage
+fi
+
+AdminPort=7225
+if [ "${2}" != "" ]; then
+    AdminPort=${2}
+fi
+
+ContainerName="md_recorder_monitor.${Instance}"
+
+if is_container_running "$ContainerName"; then
+    echo "Container ${ContainerName} is already running. Aborted."
+    exit
+fi
+
+Cmd="docker run"
+Cmd="${Cmd} -d"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --name=${ContainerName}"
+Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} cloud21.cvtt.vpn:5500/md_recorder_monitor:latest"
+Cmd="${Cmd} ${Instance} ${AdminPort}"
+
+echo ${Cmd}
+eval ${Cmd}

release_version.txt

@@ -1 +1 @@
-1.1.5
+1.2.8

scripts/alpaca_hist_md/load_rel_liq.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+export PYTHONPATH=${HOME}/prod
+
+Python=${HOME}/.pyenv/python3.10-venv/bin/python3
+Script=${HOME}/prod/cvttpy/exchanges/alpaca/hist_md/rl_calc_loader.py
+DbFile=${HOME}/prod/data/rel_liquidity.db
+Config=http://cloud23.cvtt.vpn:6789/apps/minimal_md_eqt
+
+Cmd="${Python}"
+Cmd="${Cmd} ${Script}"
+Cmd="${Cmd} --config=${Config}"
+Cmd="${Cmd} --db_file=${DbFile}"
+
+echo ${Cmd}
+eval ${Cmd}
+
+echo Done ${0} ${*}

scripts/crypto_hist_md/exchanges_trading_stats.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+export PYTHONPATH=/home/cvtt/prod
+
+Cmd="/home/cvtt/.pyenv/python3.10-venv/bin/python3"
+Cmd="${Cmd} /home/cvtt/prod/cvttpy/apps/research/exchange_trading_stats.py"
+Cmd="${Cmd} --config=http://cloud16.cvtt.vpn:6789/apps/tests/listen_market_data"
+Cmd="${Cmd} --active_exchanges=OKX,GEMINI,BITSTAMP,COINBASE_AT,BNBSPOT,KRAKEN"
+Cmd="${Cmd} --instruments=OKX:PAIR-BTC-USDT,GEMINI:PAIR-BTC-USD,BITSTAMP:PAIR-BTC-USD,COINBASE:PAIR-BTC-USD,BNBSPOT:PAIR-BTC-USDT,KRAKEN:PAIR-BTC-USD"
+Cmd="${Cmd} --db_file=/home/cvtt/prod/data/exchange_trading_stats.db"
+Cmd="${Cmd} --log_file=/home/cvtt/prod/logs/%%T.EXCHANGE_TRADING_STATS.log"
+
+echo ${Cmd}
+eval ${Cmd}