progress

release_version.txt

@@ -1 +1 @@
-0.9.6
+1.0.6

scripts/ad-hoc/rearrange_alpaca_md.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+pushd () {
+    command pushd "$@" > /dev/null
+}
+
+popd () {
+    command popd "$@" > /dev/null
+}
+
+SourceRoot=/works/cvtt/md_archive/equity/alpaca_md.2
+TargetRoot=/works/cvtt/md_archive/equity/alpaca_md.NEW
+
+mkdir -p ${TargetDir}
+
+unalias ls
+cd ${SourceRoot}
+
+for year in $(ls -d 2*)
+do
+    echo $year
+    pushd $year
+    for letter in $(ls -d ?)
+    do
+        pushd ${letter}
+
+        echo "${year}/${letter}"
+        for symbol in $(ls -d ${letter}*)
+        do
+           pushd $symbol
+
+           echo "${year}/${letter}/${symbol}/* --> ${letter}/${symbol}/${year}/"
+           mkdir -p ${TargetRoot}/${letter}/${symbol}/${year}
+           mv ${SourceRoot}/${year}/${letter}/${symbol}/* ${TargetRoot}/${letter}/${symbol}/${year}/
+
+           popd
+        done
+
+        popd
+    done
+    popd
+done

scripts/alpaca_hist_md/alpaca_md_day.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+date_to_load=${1}
+
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/alpaca_md_day #:latest
+LastDayFile=/home/cvtt/prod/logs/last_alpaca_hist_day
+ContainerName=alpaca_md_day
+
+is_container_running() {
+    local container_name=$1
+
+    if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
+        return 0  # true
+    else
+        return 1  # false
+    fi
+}
+
+
+if [ -z ${date_to_load} ] ; then
+    echo "Yesterday data has priority. Running historical container will be killed"
+    docker kill ${ContainerName}
+else
+    echo "Historical run ${date_to_load}"
+    if is_container_running "$ContainerName"; then
+        echo "Container ${ContainerName} is already running."
+        exit 3
+    fi
+    if [ "${date_to_load}" == "next" ] ; then
+        if [ ! -e ${LastDayFile} ]; then
+            echo "File ${LastDayFile} does not exist. Will try to use prev file."
+            if [ ! -e ${LastDayFile}.prev ]; then
+                echo "File ${LastDayFile}.prev does not exist. Aborted."
+                exit 2
+            fi
+            mv ${LastDayFile}.prev ${LastDayFile}
+        fi
+        last_date=$(cat ${LastDayFile} | xargs)
+        if [ -z "${last_date}" ] ; then
+            echo "File ${LastDayFile} returned an empty last day. Will try to use prev file."
+            if [ ! -e ${LastDayFile}.prev ]; then
+                echo "File ${LastDayFile}.prev does not exist. Aborted."
+                exit 2
+            fi
+            mv ${LastDayFile}.prev ${LastDayFile}
+            last_date=$(cat ${LastDayFile} | xargs)
+            if [ -z "${last_date}" ] ; then
+                echo "Unable to obtain last_date. Aborted"
+                exit 3
+            fi
+            echo "No last date_to_load in ${LastDayFile}"
+            exit 1
+        fi
+        mv ${LastDayFile} ${LastDayFile}.prev
+        date_to_load=$(date -d "${last_date} - 1 day" "+%Y-%m-%d")
+    fi
+    echo "Historical Data for ${date_to_load}"
+fi
+
+Cmd="docker run"
+Cmd="${Cmd} --pull"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ContainerName}"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${date_to_load}"
+
+echo $Cmd
+eval $Cmd
+
+if [ "$?" != "0" ] ; then
+    exit 1 # if killed we do not save last day
+fi
+
+# truncate to avoid false positive
+date_to_load=$(echo "${date_to_load} | xargs")
+if [ -n "${date_to_load}" ]; then
+    echo "Saving date_to_load to ${LastDayFile}"
+    echo ${date_to_load} >  ${LastDayFile}
+fi

scripts/crypto_hist_md/crypto_md_day.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+host=${1}
+date=${2}
+
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/crypto_md_day
+
+if [ -z ${date} ] ; then
+    date=$(date -d "yesterday" +%Y%m%d)
+fi
+
+Cmd="docker run"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=crypto_md_day.${host}.${date}"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} -h ${host}"
+Cmd="${Cmd} -d ${date}"
+Cmd="${Cmd} -s coinbase,bnbspot,bnbfut"
+
+echo $Cmd
+eval $Cmd

utils/prune_data.sh

@@ -3,14 +3,14 @@
 Src=${1}
 Days=${2}
 
-if  [ "${Src}" == "" ]
-then
-    echo "Usage: $0 <source_dir> [<num_days>]"
+if [ -z "$Days" ] || [ -z "$Src" ]; then
+    echo "Usage: $0 <source_dir> <num_days>"
     exit 1
 fi
-if  [ "${Days}" == "" ]
-then
-    Days=30
+
+# Add / if Src does not have it (symlinks)
+if [ "${Src: -1}" != "/" ]; then
+    Src="${Src}/"
 fi
 
 declare -A Settings=()
@@ -21,13 +21,13 @@ src=${Settings[Src]}
 prune_date=${Settings[PruneDate]}
 
 
-echo "Finding files older than ${prune_date}..."
+echo "Finding files older than ${prune_date} in ${Src} ..."
 Cmd="find ${src} -type f ! -newermt \"${prune_date}\""
 echo ${Cmd}
 files=($(eval ${Cmd}))
-total_files= ${#files[*]}
+total_files=${#files[*]}
 
-if [[ ${total} == 0 ]]
+if [[ ${total_files} == 0 ]]
 then
     echo "No files found to be pruned. Bye..."
     exit 0
@@ -38,11 +38,12 @@ duf ${src}
 
 echo "The following files will be removed:"
 echo "===================================="
-for f in $files ; do  echo $f; done
+for f in $files ; do  ls -l $f; done
 echo "===================================="
 echo "Total files to be pruned: ${total_files}"
 
-Cmd="${Cmd} -delete"
+echo "Removing files..."
+Cmd="${Cmd} -print -delete"
 echo ${Cmd}
 eval ${Cmd}