progress

doc/crypto_md.txt

@@ -0,0 +1,44 @@
+-------------------------------------
+C R Y P T O  M A R K E T   D A T A
+-------------------------------------
+============
+ Exchanges
+============
+	|       Coinbase
+	|       Binance Spot
+	|       Binance Futures
+	|
+ 	v
+============
+ Databases
+============
+	|       TimescaleDB: cloud21
+	|       TimescaleDB: cvttdata
+	v
+================
+ Daily Archive
+================
+    |       created by crontabs:
+    |       cvttdata:
+    |         (1) /home/cvtt/prod/utils/archive_yesterday_md.sh cloud21
+    |         (2) /home/cvtt/prod/utils/archive_yesterday_md.sh cvttdata
+    |       stored in:
+    |         cvttdata:/home/cvtt/prod/archive/md_archive/crypto/cloud21 (1)
+    |         cvttdata:/home/cvtt/prod/archive/md_archive/crypto/cvttdata (2)
+    |
+    v
+==========
+ Storage
+==========
+    |       created by crontabs:
+    |       cvttdata:
+	|              /home/cvtt/prod/utils/sync_crypto_archives.sh
+    |              TBD  prune files older than 7 days
+    |       stored in:
+    |         homestore:/works/cvtt/md_archive/crypto/  (cvttdata and cloud21)
+    |         cloud21:/opt/store/cvtt/md_archive/crypto/  (cvttdata and cloud21)
+    |
+    v
+==========
+ Usage
+==========

doc/equity_md.txt

@@ -0,0 +1,46 @@
+-------------------------------------
+E Q U I T Y  M A R K E T   D A T A
+-------------------------------------
+=====================
+ Exchanges (Sources)
+=====================
+	|       Alpaca
+	|
+    |
+    |                     TBD  ============
+    |                     TBD  Databases
+    |                     TBD  ============
+    |                     TBD      |       TimescaleDB: cloud21
+    |                     TBD      |       TimescaleDB: cvttdata
+    v                     TBD      v
+================
+ Daily Archive
+================
+    |       created by crontab on cvttdata:
+    |         /home/cvtt/prod/run/alpaca_md.sh
+    |       stored in:
+    |         cvttdata:/home/cvtt/prod/archive/md_archive/equity/alpaca_md   (TBD)
+    |
+    v
+
+
+
+
+
+
+==========
+ Storage
+==========
+    |       created by crontabs:
+    |       cvttdata:
+    |
+    |       stored in:
+    |         homestore:/works/cvtt/md_archive/crypto/cvttdata
+    |         homestore:/works/cvtt/md_archive/crypto/cloud21
+    |       backed up in:
+    |         cloud21:/opt/store/cvtt/md_archive/crypto/cvttdata
+    |         cloud21:/opt/store/cvtt/md_archive/crypto/cloud21
+    v
+==========
+ Usage
+==========

release_version.txt

@@ -1 +1 @@
-0.8.2.2
+0.9.1

scripts/alpaca_hist_md/alpaca_md.sh

@@ -48,16 +48,9 @@ export PYTHONPATH=/home/cvtt/prod
 export Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3
 export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
 export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
-export OutputDir=/home/cvtt/host_drive/alpaca_md # Local
-export LogDir=/home/cvtt/prod/logs/alpaca_md
 
-# ----- T E M P
-# export PYTHONPATH=/home/oleg/develop/cvtt2
-# export Python=/home/oleg/.pyenv/python3.10-venv/bin/python3
-# export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
-# export PyScript=/home/oleg/develop/cvtt2/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
-# export OutputDir=/home/oleg/develop/cvtt2/tmp # Local
-# export LogDir=/home/oleg/develop/cvtt2/tmp
+export OutputDir=/home/cvtt/prod/archive/md_archive/equity/alpaca_md # Local
+export LogDir=/home/cvtt/prod/logs/alpaca_md
 
 mkdir -p ${LogDir}
 

scripts/archive_yesterday_md.sh

@@ -4,14 +4,14 @@ Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3.10
 RootDir=/home/cvtt/prod
 export PYTHONPATH=${RootDir}
 
-host=$(hostname)
+host=${1}
 if [ "${host}" == "cvttdata" ]
 then
-    ArchiveRootDir=/home/cvtt/prod/archive/md_archive
+    ArchiveRootDir=/home/cvtt/prod/archive/md_archive/crypto/cvttdata
     CredKey=TSDB_MD_CVTTDATA_RO
-elif [ "${host}" == "cloud21.cryptovaltrading.com" ]
+elif [ "${host}" == "cloud21" ]
 then
-    ArchiveRootDir=/opt/store/cvtt/archive/md_archive
+    ArchiveRootDir=/home/cvtt/prod/archive/md_archive/crypto/cloud21
     CredKey=TSDB_MD_CLD21_RO
 else
     echo "Unknown host ${host}. ${0} Aborted."
@@ -21,7 +21,7 @@ fi
 mkdir -p ${ArchiveRootDir}
 
 yesterday=$(date -d "yesterday" +%Y%m%d)
-Schemas=${1}
+Schemas=${2}
 if [ "${Schemas}" == "" ]
 then
     Schemas="coinbase,bnbspot,bnbfut"

scripts/healthcheck/storage_health_check.sh

@@ -38,20 +38,13 @@ Metrics=()
 TempFiles=
 
 function cleanup {
+    echo Cleaing up temporary files: ${TempFiles}
     if [ "" != "${TempFiles}" ]; then
-        echo rm -f ${TempFiles}
         rm -f ${TempFiles}
     fi
 }
 trap cleanup EXIT
 
-
-function new_tempfile {
-    tmpfile=$(mktemp)
-    TempFiles="${TempFiles} ${tmpfile}"
-    echo ${tmpfile}
-}
-
 function space_alert() {
     ALERT_USAGE=75%
     for metric in "${Metrics[@]}"
@@ -105,8 +98,12 @@ function storage_check() {
     done
 }
 
-tmpfile=$(new_tempfile)
-tmpfile2=$(new_tempfile)
+tmpfile=$(mktemp)
+TempFiles="${TempFiles} ${tmpfile}"
+
+tmpfile2=$(mktemp)
+TempFiles="${TempFiles} ${tmpfile2}"
+
 storage_check > ${tmpfile2}
 
 echo "## :card_file_box: STORAGE HEALTH CHECK" >> ${tmpfile}
@@ -116,8 +113,9 @@ echo "| --- | --- | --- |"  >> ${tmpfile}
 cat ${tmpfile2} | sort -h -r | awk -F'%' '{printf "%s%%%s\n",$2,$3}' >> ${tmpfile}
 cat ${tmpfile} |  ${Sender} ${StatusChannel}
 
-Measurements=()
-tmpfile=$(new_tempfile)
+tmpfile=$(mktemp)
+TempFiles="${TempFiles} ${tmpfile}"
+
 space_alert > ${tmpfile}
 if [ -s ${tmpfile} ]
 then

scripts/legacy_eqt_md/process_md.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Collect all "A" files
+letter=${1}
+
+if [ "${letter}" == "" ]; then
+	echo "Usage $0 <letter>"
+	exit 1
+fi
+
+# copy text data to local location
+rsync -ahv /home/cvtt/host_drive/eqt_hist_md/txt/${letter} /home/cvtt/tmp/txt/
+
+# create db files
+PYTHONPATH=/home/cvtt/prod /home/cvtt/.pyenv/python3.10-venv/bin/python3 /home/cvtt/prod/cvttpy/trading/mkt_data/ad_hoc/eqt_md_to_db.py /home/cvtt/tmp/txt/${letter} /home/cvtt/tmp/db
+
+if [ "$?" != "0" ] ; then
+    exit
+fi
+
+# Move all files to host drive
+# a) create file list
+(cd /home/cvtt/tmp/db/ && find . -name '*db' -print | grep "/${letter}/") > /home/cvtt/tmp/tran_db/${letter}_files
+
+echo ${letter} is done
+exit
+
+
+# b) rsync files to host drive
+rsync -ahv --remove-source-files --files-from=/home/cvtt/tmp/tran_db/${letter}_files /home/cvtt/tmp/db/ cvtt@my-vm-host:/localdisk/cvtt/eqt_hist_md/db/
+
+# Clean directories
+( cd /home/cvtt/tmp/db && (for d in $(find . -name $letter -type d -print); do echo $d ; done) | grep -v /$letter/$letter | xargs rm -rf) && rm -rf /home/cvtt/tmp/txt/${letter}