progress

release_version.txt

@@ -1 +1 @@
-0.9.6
+1.0.3

scripts/ad-hoc/rearrange_alpaca_md.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+pushd () {
+    command pushd "$@" > /dev/null
+}
+
+popd () {
+    command popd "$@" > /dev/null
+}
+
+SourceRoot=/works/cvtt/md_archive/equity/alpaca_md.2
+TargetRoot=/works/cvtt/md_archive/equity/alpaca_md.NEW
+
+mkdir -p ${TargetDir}
+
+unalias ls
+cd ${SourceRoot}
+
+for year in $(ls -d 2*)
+do
+    echo $year
+    pushd $year
+    for letter in $(ls -d ?)
+    do
+        pushd ${letter}
+
+        echo "${year}/${letter}"
+        for symbol in $(ls -d ${letter}*)
+        do
+           pushd $symbol
+
+           echo "${year}/${letter}/${symbol}/* --> ${letter}/${symbol}/${year}/"
+           mkdir -p ${TargetRoot}/${letter}/${symbol}/${year}
+           mv ${SourceRoot}/${year}/${letter}/${symbol}/* ${TargetRoot}/${letter}/${symbol}/${year}/
+
+           popd
+        done
+
+        popd
+    done
+    popd
+done

scripts/alpaca_hist_md/alpaca_md_day.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+date_to_load=${1}
+
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/alpaca_md_day:latest
+LastDayFile=/home/cvtt/prod/logs/last_alpaca_hist_day
+
+if [ -z ${date_to_load} ] ; then
+    echo "Yesterday data. Historical container will be killed"
+    docker kill alpaca_md_day
+else
+    if [ "${date_to_load}" == "next" ] ; then
+        last_date=$(cat ${LastDayFile})
+        if [ -z ${last_date} ] ; then
+            echo "No last date_to_load in ${LastDayFile}"
+            exit 1
+        fi
+        date_to_load=$(date -d "${last_date} - 1 day" "+%Y-%m-%d")
+    fi
+    echo "Historical Data for ${date_to_load}"
+fi
+
+Cmd="docker run"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=alpaca_md_day"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${date_to_load}"
+
+echo $Cmd
+eval $Cmd
+
+if [ "$?" != "0" ] ; then
+    exit 1 # if killed we do not save last day
+fi
+
+if [ -n ${date_to_load}]; then
+    echo "Saving date_to_load to ${LastDayFile}"
+    echo ${date_to_load} >  ${LastDayFile}
+fi

scripts/crypto_hist_md/crypto_md_day.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+host=${1}
+date=${2}
+
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/crypto_md_day
+
+if [ -z ${date} ] ; then
+    date=$(date -d "yesterday" +%Y%m%d)
+fi
+
+Cmd="docker run"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=crypto_md_day.${host}.${date}"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} -h ${host}"
+Cmd="${Cmd} -d ${date}"
+Cmd="${Cmd} -s coinbase,bnbspot,bnbfut"
+
+echo $Cmd
+eval $Cmd

utils/prune_data.sh

@@ -3,14 +3,14 @@
 Src=${1}
 Days=${2}
 
-if  [ "${Src}" == "" ]
+if [ -z "$Days" ] || [ -z "$Src" ]; then
-then
+    echo "Usage: $0 <source_dir> <num_days>"
-    echo "Usage: $0 <source_dir> [<num_days>]"
     exit 1
 fi
-if  [ "${Days}" == "" ]
+
-then
+# Add / if Src does not have it (symlinks)
-    Days=30
+if [ "${Src: -1}" != "/" ]; then
+    Src="${Src}/"
 fi
 
 declare -A Settings=()
@@ -21,13 +21,13 @@ src=${Settings[Src]}
 prune_date=${Settings[PruneDate]}
 
 
-echo "Finding files older than ${prune_date}..."
+echo "Finding files older than ${prune_date} in ${Src} ..."
 Cmd="find ${src} -type f ! -newermt \"${prune_date}\""
 echo ${Cmd}
 files=($(eval ${Cmd}))
-total_files= ${#files[*]}
+total_files=${#files[*]}
 
-if [[ ${total} == 0 ]]
+if [[ ${total_files} == 0 ]]
 then
     echo "No files found to be pruned. Bye..."
     exit 0
@@ -38,11 +38,12 @@ duf ${src}
 
 echo "The following files will be removed:"
 echo "===================================="
-for f in $files ; do  echo $f; done
+for f in $files ; do  ls -l $f; done
 echo "===================================="
 echo "Total files to be pruned: ${total_files}"
 
-Cmd="${Cmd} -delete"
+echo "Removing files..."
+Cmd="${Cmd} -print -delete"
 echo ${Cmd}
 eval ${Cmd}