progress

doc/crypto_md.txt

@@ -30,12 +30,11 @@ C R Y P T O  M A R K E T   D A T A
 ==========
  Storage
 ==========
-    |       created by crontabs:
+    |       created by crontab cvttdata:
-    |       cvttdata:
+    |           /home/cvtt/prod/utils/sync_market_data.sh (ops/utls)
-    |              TBD
     |       stored in:
-    |         homestore:/works/cvtt/md_archive/crypto/  (cvttdata and cloud21)
+    |         homestore:/works/cvtt/md_archive/equity/alpaca_md
-    |         cloud21:/opt/store/cvtt/md_archive/crypto/  (cvttdata and cloud21)
+    |         cloud21:/opt/store/cvtt/md_archive/equity/alpaca_md
     |
     v
 ==========

doc/equity_md.txt

@@ -19,28 +19,15 @@ E Q U I T Y  M A R K E T   D A T A
     |       created by crontab on cvttdata:
     |         /home/cvtt/prod/run/alpaca_md.sh
     |       stored in:
-    |         cvttdata:/home/cvtt/prod/archive/md_archive/equities/alpaca_md   (TBD)
+    |         cvttdata:/home/cvtt/prod/archive/md_archive/equity/alpaca_md 
     |
     v
-
-
-
-
-
-    
 ==========
  Storage
 ==========
-    |       created by crontabs:
+    |       created by crontab cvttdata:
-    |       cvttdata:
+    |           /home/cvtt/prod/utils/sync_market_data.sh
-    |
     |       stored in:
-    |         homestore:/works/cvtt/md_archive/crypto/cvttdata
+    |         homestore:/works/cvtt/md_archive/equity/alpaca_md
-    |         homestore:/works/cvtt/md_archive/crypto/cloud21
+    |         cloud21:/opt/store/cvtt/md_archive/equity/alpaca_md
-    |       backed up in:
+____V_______
-    |         cloud21:/opt/store/cvtt/md_archive/crypto/cvttdata
-    |         cloud21:/opt/store/cvtt/md_archive/crypto/cloud21
-    v
-==========
- Usage
-==========

release_version.txt

@@ -1 +1 @@
-0.8.9
+0.9.4

scripts/alpaca_hist_md/alpaca_md.sh

@@ -49,17 +49,9 @@ export Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3
 export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
 export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
 
-export OutputDir=/home/cvtt/prod/archive/md_archive/equities/alpaca_md # Local
+export OutputDir=/home/cvtt/prod/archive/md_archive/equity/alpaca_md # Local
 export LogDir=/home/cvtt/prod/logs/alpaca_md
 
-# ----- T E M P
-# export PYTHONPATH=/home/oleg/develop/cvtt2
-# export Python=/home/oleg/.pyenv/python3.10-venv/bin/python3
-# export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
-# export PyScript=/home/oleg/develop/cvtt2/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
-# export OutputDir=/home/oleg/develop/cvtt2/tmp # Local
-# export LogDir=/home/oleg/develop/cvtt2/tmp
-
 mkdir -p ${LogDir}
 
 run_proc() {
@@ -113,3 +105,10 @@ for ((ii=0; ii <${#Instruments[@]}; ii+=slice_size)); do
 
     parallel -j ${NumJobs} run_proc {} ${Start} ${End} ::: "${InstSlice[@]}"
 done
+
+echo "Compressing"
+for file in $(find ${OutputDir} -type f -name '*db' -print )
+do
+    echo "Compressing ${file}"
+    gzip ${file}
+done

utils/sync_market_data.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+Source=/home/cvtt/prod/archive/md_archive/
+Targets=
+Targets="${Targets} oleg@homestore.cvtt.vpn:/works/cvtt/md_archive/"
+Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/"
+
+
+for tgt in ${Targets}
+do
+    Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
+    echo $Cmd
+    eval $Cmd
+done