fix

doc/crypto_md.txt

@@ -0,0 +1,42 @@
+-------------------------------------
+C R Y P T O  M A R K E T   D A T A
+-------------------------------------
+============
+ Exchanges
+============
+	|       Coinbase
+	|       Binance Spot
+	|       Binance Futures
+	|
+ 	v
+============
+ Databases
+============
+	|       TimescaleDB: cloud21
+	|       TimescaleDB: cvttdata
+	v
+================
+ Daily Archive
+================
+    |       created by crontabs:
+    |       cvttdata:
+    |         (1) /home/cvtt/prod/utils/archive_yesterday_md.sh cloud21
+    |         (2) /home/cvtt/prod/utils/archive_yesterday_md.sh cvttdata
+    |       stored in:
+    |         cvttdata:/home/cvtt/prod/archive/md_archive/crypto/cloud21 (1)
+    |         cvttdata:/home/cvtt/prod/archive/md_archive/crypto/cvttdata (2)
+    |
+    v
+==========
+ Storage
+==========
+    |       created by crontab cvttdata:
+    |           /home/cvtt/prod/utils/sync_market_data.sh (ops/utls)
+    |       stored in:
+    |         homestore:/works/cvtt/md_archive/equity/alpaca_md
+    |         cloud21:/opt/store/cvtt/md_archive/equity/alpaca_md
+    |
+    v
+==========
+ Usage
+==========

doc/equity_md.txt

@@ -0,0 +1,33 @@
+-------------------------------------
+E Q U I T Y  M A R K E T   D A T A
+-------------------------------------
+=====================
+ Exchanges (Sources)
+=====================
+	|       Alpaca
+	|
+    |
+    |                     TBD  ============
+    |                     TBD  Databases
+    |                     TBD  ============
+    |                     TBD      |       TimescaleDB: cloud21
+    |                     TBD      |       TimescaleDB: cvttdata
+    v                     TBD      v
+================
+ Daily Archive
+================
+    |       created by crontab on cvttdata:
+    |         /home/cvtt/prod/run/alpaca_md.sh
+    |       stored in:
+    |         cvttdata:/home/cvtt/prod/archive/md_archive/equity/alpaca_md 
+    |
+    v
+==========
+ Storage
+==========
+    |       created by crontab cvttdata:
+    |           /home/cvtt/prod/utils/sync_market_data.sh
+    |       stored in:
+    |         homestore:/works/cvtt/md_archive/equity/alpaca_md
+    |         cloud21:/opt/store/cvtt/md_archive/equity/alpaca_md
+____V_______

docker/alpaca_md_day.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+
+date_to_load=${1}
+
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/alpaca_md_day #:latest
+LastDayFile=/home/cvtt/prod/logs/last_alpaca_hist_day
+ContainerName=alpaca_md_day
+
+is_container_running() {
+    local container_name=$1
+
+    if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
+        return 0  # true
+    else
+        return 1  # false
+    fi
+}
+
+export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+
+is_business_day() {
+    dt=${1}
+
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+get_prev_business_day() {
+    Start=${1}
+    while true; do
+        if is_business_day ${Start}; then
+            break
+        fi
+        echo "${Start} is not business day in US" >&2
+        Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
+    done
+    echo ${Start}
+}
+export -f get_prev_business_day
+
+if [ -z ${date_to_load} ] ; then
+    echo "Yesterday data has priority. Running historical container will be stopped"
+    Cmd="docker stop ${ContainerName}"
+    echo ${Cmd} && eval ${Cmd}
+    Cmd="docker kill ${ContainerName}"
+    echo ${Cmd} && eval ${Cmd}
+    Cmd="docker rm -f ${ContainerName}"
+    echo ${Cmd} && eval ${Cmd}
+
+    Cmd="docker ps"
+    echo ${Cmd} && eval ${Cmd}
+
+    if is_container_running "$ContainerName"; then
+        echo "Container ${ContainerName} is still running."
+        exit 3
+    fi
+else
+    echo "Historical run ${date_to_load}"
+    if is_container_running "$ContainerName"; then
+        echo "Container ${ContainerName} is already running."
+        exit 3
+    fi
+    if [ "${date_to_load}" == "next" ] ; then
+        if [ ! -e ${LastDayFile} ]; then
+            echo "File ${LastDayFile} does not exist. Will try to use prev file."
+            if [ ! -e ${LastDayFile}.prev ]; then
+                echo "File ${LastDayFile}.prev does not exist. Aborted."
+                exit 2
+            fi
+            mv ${LastDayFile}.prev ${LastDayFile}
+        fi
+        last_date=$(cat ${LastDayFile} | xargs)
+        if [ -z "${last_date}" ] ; then
+            echo "File ${LastDayFile} returned an empty last day. Will try to use prev file."
+            if [ ! -e ${LastDayFile}.prev ]; then
+                echo "File ${LastDayFile}.prev does not exist. Aborted."
+                exit 2
+            fi
+            mv ${LastDayFile}.prev ${LastDayFile}
+            last_date=$(cat ${LastDayFile} | xargs)
+            if [ -z "${last_date}" ] ; then
+                echo "Unable to obtain last_date. Aborted"
+                exit 3
+            fi
+            echo "No last date_to_load in ${LastDayFile}"
+            exit 1
+        fi
+        mv ${LastDayFile} ${LastDayFile}.prev
+        date_to_load=$(date -d "${last_date} - 1 day" "+%Y-%m-%d")
+    fi
+fi
+if [ -n "${date_to_load}" ]; then
+    date_to_load=$(get_prev_business_day ${date_to_load})
+    echo "Historical Data for ${date_to_load}"
+fi
+
+Cmd="docker run"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ContainerName}"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${date_to_load}"
+
+echo $Cmd
+eval $Cmd
+
+if [ "$?" != "0" ] ; then
+    exit 1 # if killed we do not save last day
+fi
+
+# truncate to avoid false positive
+date_to_load=$(echo "${date_to_load}" | xargs)
+if [ -n "${date_to_load}" ]; then
+    echo "Saving date_to_load to ${LastDayFile}"
+    echo ${date_to_load} >  ${LastDayFile}
+fi

docker/config_service.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+Cmd="docker run"
+Cmd="${Cmd} -d"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --name=cvtt_config_service"
+Cmd="${Cmd} -p 6789:6789"
+Cmd="${Cmd} -v /home/cvtt/prod/config_service/data:/app/data"
+Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} cloud21.cvtt.vpn:5500/config_service:latest"
+
+echo ${Cmd}
+eval ${Cmd}

docker/crypto_md_day.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+host=${1}
+date=${2}
+
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/crypto_md_day
+
+if [ -z ${date} ] ; then
+    date=$(date -d "yesterday" +%Y%m%d)
+fi
+
+Cmd="docker run"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=crypto_md_day.${host}.${date}"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} -h ${host}"
+Cmd="${Cmd} -d ${date}"
+Cmd="${Cmd} -s coinbase,bnbspot,bnbfut"
+
+echo $Cmd
+eval $Cmd

docker/md_recorder.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# runs on host to start container
+usage() {
+  echo "Usage: $0 <job_name> (BNBFUT_CLD21, BNBSPOT_CLD21, BNBSPOT_CVTTDATA, COINBASE_CLD21)"
+  exit 1
+}
+
+
+job=${1}
+if [ "${job}" == "" ] ; then
+    usage
+fi
+
+Cmd="docker run"
+Cmd="${Cmd} -d"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --name=md_recorder.${job}"
+Cmd="${Cmd} -v /home/cvtt/prod/data:/app/data"
+Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} cloud21.cvtt.vpn:5500/md_recorder:latest"
+Cmd="${Cmd} ${1}"
+
+echo ${Cmd}
+eval ${Cmd}

docker/relative_liquidity.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+Cmd="docker run"
+Cmd="${Cmd} -d"
+#Cmd="${Cmd} --rm"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --name=relative_liquidity_svc"
+Cmd="${Cmd} -p 5678:5678"
+Cmd="${Cmd} -v /home/cvtt/prod/data:/app/data"
+Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} cloud21.cvtt.vpn:5500/relative_liquidity:latest"
+
+echo ${Cmd}
+eval ${Cmd}

docker/trading_calendar.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+#Cmd="docker pull apptasticsoftware/trading-calendar:latest"
+#echo ${Cmd} && eval ${Cmd}
+
+Cmd="docker run"
+Cmd="${Cmd} -d"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --name trading-calendar"
+Cmd="${Cmd} -p 8000:80 apptasticsoftware/trading-calendar"
+echo ${Cmd} && eval ${Cmd}

release_version.txt

@@ -1 +1 @@
-0.8.2.1
+1.1.6

scripts/ad-hoc/rearrange_alpaca_md.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+pushd () {
+    command pushd "$@" > /dev/null
+}
+
+popd () {
+    command popd "$@" > /dev/null
+}
+
+SourceRoot=/works/cvtt/md_archive/equity/alpaca_md.2
+TargetRoot=/works/cvtt/md_archive/equity/alpaca_md.NEW
+
+mkdir -p ${TargetDir}
+
+unalias ls
+cd ${SourceRoot}
+
+for year in $(ls -d 2*)
+do
+    echo $year
+    pushd $year
+    for letter in $(ls -d ?)
+    do
+        pushd ${letter}
+
+        echo "${year}/${letter}"
+        for symbol in $(ls -d ${letter}*)
+        do
+           pushd $symbol
+
+           echo "${year}/${letter}/${symbol}/* --> ${letter}/${symbol}/${year}/"
+           mkdir -p ${TargetRoot}/${letter}/${symbol}/${year}
+           mv ${SourceRoot}/${year}/${letter}/${symbol}/* ${TargetRoot}/${letter}/${symbol}/${year}/
+
+           popd
+        done
+
+        popd
+    done
+    popd
+done

scripts/alpaca_hist_md/alpaca_md.sh

@@ -48,16 +48,9 @@ export PYTHONPATH=/home/cvtt/prod
 export Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3
 export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
 export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
-export OutputDir=/home/cvtt/host_drive/alpaca_md # Local
-export LogDir=/home/cvtt/prod/logs/alpaca_md
 
-# ----- T E M P
-# export PYTHONPATH=/home/oleg/develop/cvtt2
-# export Python=/home/oleg/.pyenv/python3.10-venv/bin/python3
-# export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
-# export PyScript=/home/oleg/develop/cvtt2/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
-# export OutputDir=/home/oleg/develop/cvtt2/tmp # Local
-# export LogDir=/home/oleg/develop/cvtt2/tmp
+export OutputDir=/home/cvtt/prod/archive/md_archive/equity/alpaca_md # Local
+export LogDir=/home/cvtt/prod/logs/alpaca_md
 
 mkdir -p ${LogDir}
 
@@ -112,3 +105,10 @@ for ((ii=0; ii <${#Instruments[@]}; ii+=slice_size)); do
 
     parallel -j ${NumJobs} run_proc {} ${Start} ${End} ::: "${InstSlice[@]}"
 done
+
+echo "Compressing"
+for file in $(find ${OutputDir} -type f -name '*db' -print )
+do
+    echo "Compressing ${file}"
+    gzip ${file}
+done

scripts/archive_yesterday_md.sh

@@ -4,14 +4,14 @@ Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3.10
 RootDir=/home/cvtt/prod
 export PYTHONPATH=${RootDir}
 
-host=$(hostname)
+host=${1}
 if [ "${host}" == "cvttdata" ]
 then
-    ArchiveRootDir=/home/cvtt/prod/archive/md_archive
+    ArchiveRootDir=/home/cvtt/prod/archive/md_archive/crypto/cvttdata
     CredKey=TSDB_MD_CVTTDATA_RO
-elif [ "${host}" == "cloud21.cryptovaltrading.com" ]
+elif [ "${host}" == "cloud21" ]
 then
-    ArchiveRootDir=/opt/store/cvtt/archive/md_archive
+    ArchiveRootDir=/home/cvtt/prod/archive/md_archive/crypto/cloud21
     CredKey=TSDB_MD_CLD21_RO
 else
     echo "Unknown host ${host}. ${0} Aborted."
@@ -21,7 +21,7 @@ fi
 mkdir -p ${ArchiveRootDir}
 
 yesterday=$(date -d "yesterday" +%Y%m%d)
-Schemas=${1}
+Schemas=${2}
 if [ "${Schemas}" == "" ]
 then
     Schemas="coinbase,bnbspot,bnbfut"

scripts/build_release.sh

@@ -7,25 +7,46 @@ git_repo_arr[cvttpy]=git@cloud21.cvtt.vpn:/opt/store/git/cvttpy.git
 git_repo_arr[ops]=git@cloud21.cvtt.vpn:/opt/store/git/ops.git
 git_repo_arr[research]=git@cloud21.cvtt.vpn:/opt/store/git/research.git
 
-default_project=cvttpy
-default_branch=master
-
-
 dist_root=/home/cvttdist/software/cvtt2
 dist_user=cvttdist
 dist_host="cloud21.cvtt.vpn"
 dist_ssh_port="22"
 
 dist_locations="cloud21.cvtt.vpn:22 homestore.cvtt.vpn:22"
+# ---------------- Settings
 
-interactive=Y
+# ---------------- cmdline
+prj=
+brnch=master
+interactive=N
 
-# cmdline
-prj=${1:-${default_project}}
-brnch=${2:-${default_branch}}
-if [ "${3}" == "D" ]; then
-    interactive=N
-fi
+usage() {
+  echo "Usage: $0 -p <project> [-b <branch (master)> -i (interactive)"
+  exit 1
+}
+
+while getopts ":p:b:i" opt; do
+  case ${opt} in
+    p )
+      prj=$OPTARG
+      ;;
+    b )
+      brnch=$OPTARG
+      ;;
+    i )
+      interactive=Y
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+# ---------------- cmdline
 
 function confirm {
     if [ "${interactive}" == "Y" ]; then

scripts/healthcheck/storage_health_check.sh

@@ -38,19 +38,13 @@ Metrics=()
 TempFiles=
 
 function cleanup {
+    echo Cleaing up temporary files: ${TempFiles}
     if [ "" != "${TempFiles}" ]; then
         rm -f ${TempFiles}
     fi
 }
 trap cleanup EXIT
 
-
-function new_tempfile {
-    tmpfile=$(mktemp)
-    TempFiles="${TempFiles} ${tmpfile}"
-    echo ${tmpfile}
-}
-
 function space_alert() {
     ALERT_USAGE=75%
     for metric in "${Metrics[@]}"
@@ -104,8 +98,12 @@ function storage_check() {
     done
 }
 
-tmpfile=$(new_tempfile)
-tmpfile2=$(new_tempfile)
+tmpfile=$(mktemp)
+TempFiles="${TempFiles} ${tmpfile}"
+
+tmpfile2=$(mktemp)
+TempFiles="${TempFiles} ${tmpfile2}"
+
 storage_check > ${tmpfile2}
 
 echo "## :card_file_box: STORAGE HEALTH CHECK" >> ${tmpfile}
@@ -115,14 +113,14 @@ echo "| --- | --- | --- |"  >> ${tmpfile}
 cat ${tmpfile2} | sort -h -r | awk -F'%' '{printf "%s%%%s\n",$2,$3}' >> ${tmpfile}
 cat ${tmpfile} |  ${Sender} ${StatusChannel}
 
-Measurements=()
-tmpfile=$(new_tempfile)
+tmpfile=$(mktemp)
+TempFiles="${TempFiles} ${tmpfile}"
+
 space_alert > ${tmpfile}
-cat ${tmpfile}
 if [ -s ${tmpfile} ]
 then
     (echo "### :card_file_box: STORAGE ALERTS" && cat ${tmpfile}) | ${Sender} ${AlertChannel}
 else
-    echo File ${tmpfile} is empty
+    echo "No Storage Alerts"
 fi
 

scripts/legacy_eqt_md/process_md.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Collect all "A" files
+letter=${1}
+
+if [ "${letter}" == "" ]; then
+	echo "Usage $0 <letter>"
+	exit 1
+fi
+
+# copy text data to local location
+rsync -ahv /home/cvtt/host_drive/eqt_hist_md/txt/${letter} /home/cvtt/tmp/txt/
+
+# create db files
+PYTHONPATH=/home/cvtt/prod /home/cvtt/.pyenv/python3.10-venv/bin/python3 /home/cvtt/prod/cvttpy/trading/mkt_data/ad_hoc/eqt_md_to_db.py /home/cvtt/tmp/txt/${letter} /home/cvtt/tmp/db
+
+if [ "$?" != "0" ] ; then
+    exit
+fi
+
+# Move all files to host drive
+# a) create file list
+(cd /home/cvtt/tmp/db/ && find . -name '*db' -print | grep "/${letter}/") > /home/cvtt/tmp/tran_db/${letter}_files
+
+echo ${letter} is done
+exit
+
+
+# b) rsync files to host drive
+rsync -ahv --remove-source-files --files-from=/home/cvtt/tmp/tran_db/${letter}_files /home/cvtt/tmp/db/ cvtt@my-vm-host:/localdisk/cvtt/eqt_hist_md/db/
+
+# Clean directories
+( cd /home/cvtt/tmp/db && (for d in $(find . -name $letter -type d -print); do echo $d ; done) | grep -v /$letter/$letter | xargs rm -rf) && rm -rf /home/cvtt/tmp/txt/${letter}

utils/prune_data.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+Src=${1}
+Days=${2}
+
+if [ -z "$Days" ] || [ -z "$Src" ]; then
+    echo "Usage: $0 <source_dir> <num_days>"
+    exit 1
+fi
+
+# Add / if Src does not have it (symlinks)
+if [ "${Src: -1}" != "/" ]; then
+    Src="${Src}/"
+fi
+
+declare -A Settings=()
+Settings[Src]=${Src}
+Settings[PruneDate]=$(date -d "${Days} days ago" '+%Y-%m-%d')
+
+src=${Settings[Src]}
+prune_date=${Settings[PruneDate]}
+
+
+echo "Finding files older than ${prune_date} in ${Src} ..."
+Cmd="find ${src} -type f ! -newermt \"${prune_date}\""
+echo ${Cmd}
+files=($(eval ${Cmd}))
+total_files=${#files[*]}
+
+if [[ ${total_files} == 0 ]]
+then
+    echo "No files found to be pruned. Bye..."
+    exit 0
+fi
+
+echo Before Pruning....
+duf ${src}
+
+echo "The following files will be removed:"
+echo "===================================="
+for f in $files ; do  ls -l $f; done
+echo "===================================="
+echo "Total files to be pruned: ${total_files}"
+
+echo "Removing files..."
+Cmd="${Cmd} -print -delete"
+echo ${Cmd}
+eval ${Cmd}
+
+echo "Removing empty directories..."
+Cmd="find ${src} -type d -empty -print -delete"
+echo ${Cmd}
+eval ${Cmd}
+
+echo After Pruning....
+duf ${src}
+
+echo $0 Done

utils/sync_market_data.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+Source=/home/cvtt/prod/archive/md_archive/
+Targets=
+Targets="${Targets} oleg@homestore.cvtt.vpn:/works/cvtt/md_archive/"
+Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/"
+
+
+for tgt in ${Targets}
+do
+    Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
+    echo $Cmd
+    eval $Cmd
+done