progress

doc/crypto_md.txt

@@ -30,13 +30,11 @@ C R Y P T O  M A R K E T   D A T A
 ==========
  Storage
 ==========
-    |       created by crontabs:
+    |       created by crontab cvttdata:
-    |       cvttdata:
+    |           /home/cvtt/prod/utils/sync_market_data.sh (ops/utls)
-	|              /home/cvtt/prod/utils/sync_crypto_archives.sh
-    |              TBD  prune files older than 7 days
     |       stored in:
-    |         homestore:/works/cvtt/md_archive/crypto/  (cvttdata and cloud21)
+    |         homestore:/works/cvtt/md_archive/equity/alpaca_md
-    |         cloud21:/opt/store/cvtt/md_archive/crypto/  (cvttdata and cloud21)
+    |         cloud21:/opt/store/cvtt/md_archive/equity/alpaca_md
     |
     v
 ==========

doc/equity_md.txt

@@ -19,28 +19,15 @@ E Q U I T Y  M A R K E T   D A T A
     |       created by crontab on cvttdata:
     |         /home/cvtt/prod/run/alpaca_md.sh
     |       stored in:
-    |         cvttdata:/home/cvtt/prod/archive/md_archive/equity/alpaca_md   (TBD)
+    |         cvttdata:/home/cvtt/prod/archive/md_archive/equity/alpaca_md 
     |
     v
-
-
-
-
-
-
 ==========
  Storage
 ==========
-    |       created by crontabs:
+    |       created by crontab cvttdata:
-    |       cvttdata:
+    |           /home/cvtt/prod/utils/sync_market_data.sh
-    |
     |       stored in:
-    |         homestore:/works/cvtt/md_archive/crypto/cvttdata
+    |         homestore:/works/cvtt/md_archive/equity/alpaca_md
-    |         homestore:/works/cvtt/md_archive/crypto/cloud21
+    |         cloud21:/opt/store/cvtt/md_archive/equity/alpaca_md
-    |       backed up in:
+____V_______
-    |         cloud21:/opt/store/cvtt/md_archive/crypto/cvttdata
-    |         cloud21:/opt/store/cvtt/md_archive/crypto/cloud21
-    v
-==========
- Usage
-==========

release_version.txt

@@ -1 +1 @@
-0.9.1
+0.9.5

scripts/alpaca_hist_md/alpaca_md.sh

@@ -105,3 +105,10 @@ for ((ii=0; ii <${#Instruments[@]}; ii+=slice_size)); do
 
     parallel -j ${NumJobs} run_proc {} ${Start} ${End} ::: "${InstSlice[@]}"
 done
+
+echo "Compressing"
+for file in $(find ${OutputDir} -type f -name '*db' -print )
+do
+    echo "Compressing ${file}"
+    gzip ${file}
+done

utils/prune_data.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+Src=${1}
+Days=${2}
+
+if  [ "${Src}" == "" ]
+then
+    echo "Usage: $0 <source_dir> [<num_days>]"
+    exit 1
+fi
+if  [ "${Days}" == "" ]
+then
+    Days=30
+fi
+
+declare -A Settings=()
+Settings[Src]=${Src}
+Settings[PruneDate]=$(date -d "${Days} days ago" '+%Y-%m-%d')
+
+src=${Settings[Src]}
+prune_date=${Settings[PruneDate]}
+
+echo Before Pruning....
+duf ${src}
+
+echo "Finding files older than ${prune_date}..."
+Cmd="find ${src} -type f ! -newermt \"${prune_date}\""
+echo ${Cmd}
+files=($(eval ${Cmd}))
+echo "The following files will be removed:"
+echo "===================================="
+for f in $files ; do  echo $f; done
+echo "===================================="
+echo "Total files to be pruned: ${#files[*]}"
+
+Cmd="${Cmd} -delete"
+echo ${Cmd}
+eval ${Cmd}
+
+echo "Removing empty directories..."
+Cmd="find ${src} -type d -empty -print -delete"
+echo ${Cmd}
+eval ${Cmd}
+
+echo After Pruning....
+duf ${src}
+
+echo $0 Done

utils/sync_market_data.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+Source=/home/cvtt/prod/archive/md_archive/
+Targets=
+Targets="${Targets} oleg@homestore.cvtt.vpn:/works/cvtt/md_archive/"
+Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/"
+
+
+for tgt in ${Targets}
+do
+    Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
+    echo $Cmd
+    eval $Cmd
+done