progress

docker/crypto_md_day.sh

@@ -14,7 +14,7 @@ Cmd="docker run"
 Cmd="${Cmd} --pull=always"
 Cmd="${Cmd} --network=host"
 Cmd="${Cmd} --name=crypto_md_day.${host}.${date}"
-# Cmd="${Cmd} --rm"
+Cmd="${Cmd} --rm"
 Cmd="${Cmd} ${DockerImage}"
 Cmd="${Cmd} -h ${host}"
 Cmd="${Cmd} -d ${date}"

docker/md_portal/start_docker_md_portal.sh

@@ -48,7 +48,7 @@ if [ -z ${date} ] ; then
 fi
 
 Cmd="docker run"
-# Cmd="${Cmd} --rm"
+Cmd="${Cmd} --detach"
 Cmd="${Cmd} --restart=unless-stopped"
 Cmd="${Cmd} --pull=always"
 Cmd="${Cmd} --network=host"

release_version.txt

@@ -1 +1 @@
-1.4.4,start md_portal with command line validation
+1.4.6,removing containers for crypto_md_day

research/load_crypto_md.sh

@@ -126,6 +126,5 @@ function download_file {
     chmod 600 ${TargetFilePath}
     ls -lh ${TargetFilePath}
 
-
 }
 download_file

research/load_eqty_md.sh

@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+
+usage() {
+  echo "Usage: $0 [-h <host (homestore*/cloud21)>] [-d <YYYYMMDD> (yesterday*)] [-s <stocks comma separated>"
+  exit 1
+}
+if [[ "$(uname)" == "Darwin" ]]; then
+    # macOS
+    date='gdate'
+else
+    date='date'
+fi
+
+
+is_valid() {
+  local inst=$1
+  shift
+  local valid_instances=("$@")
+
+  for valid_inst in "${valid_instances[@]}";
+  do
+    if [[ "$inst" == "$valid_inst" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+md_date=""
+host=homestore
+stocks=COIN,GBTC,SQ
+
+while getopts ":h:d:s:" opt; do
+  case ${opt} in
+    d )
+      md_date=$OPTARG
+      ;;
+    h )
+      host=$OPTARG
+      ;;
+    s )
+      stocks=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+if [ "${md_date}" == "" ] ; then
+    md_date="yesterday"
+fi
+
+echo "$md_date $host $source"
+
+valid_hosts=('homestore' 'cloud21')
+if ! is_valid "${host}"  "${valid_hosts[@]}" ; then
+    echo "Host '${host}' is not valid"
+    usage
+fi
+
+# ---- temp dir
+echo ${SourceFile}
+tmp_dir=$(mktemp -d)
+
+function cleanup {
+    cd ${HOME}
+    rm -rf ${tmp_dir}
+}
+trap cleanup EXIT
+# ---- temp dir
+
+
+if [ "${host}" == "cloud21" ] ; then
+    SourceHost=cloud21.cvtt.vpn
+    SourceUser=cvtt
+    SourceRootDir="/opt/store/cvtt/md_archive/equity/alpaca_md"
+elif [ "${host}" == "homestore" ]; then
+    SourceHost=homestore.cvtt.vpn
+    SourceUser=oleg
+    SourceRootDir="/works/cvtt/md_archive/equity/alpaca_md"
+else
+    usage
+fi
+
+
+OLDIFS=$IFS
+IFS=','
+read -ra Stocks <<< "$stocks"
+IFS=$OLDIFS
+
+
+echo "${SourceHost} ${SourceUser} ${SourceRootDir} ${Stocks[@]}"
+
+
+function download_file {
+    local SourceFilePath=${1}
+
+    Cmd="rsync"
+    Cmd="${Cmd} -ahv"
+    if tty -s; then
+        Cmd="${Cmd} --progress=info2"
+    fi
+    Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/"
+    echo ${Cmd}
+    eval ${Cmd}
+    ls -lh ${tmp_dir}
+    Cmd="gunzip ${tmp_dir}/${SourceFileZip}"
+    echo ${Cmd} && eval ${Cmd}
+    
+    ls -lh ${tmp_dir}
+    
+    Cmd="sqlite3 ${tmp_dir}/${SourceFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}"
+    echo ${Cmd}
+    eval ${Cmd}
+
+}
+
+# TargetDir="/opt/jupyter_gpu/data/eqty_md"
+TargetDir="/tmp/aaa"
+mkdir -p ${TargetDir}
+
+TargetFile=$(${date} -d ${md_date} "+%Y%m%d.eqty.mktdata.ohlcv.db")
+TargetFilePath="${TargetDir}/${TargetFile}"
+rm -f ${TargetFilePath}
+touch ${TargetFilePath}
+
+Table=md_1min_bars
+
+echo "Loading files"
+for stock in "${Stocks[@]}"; do
+    StockLetter="${stock:0:1}"
+    SourceFile=$(${date} -d ${md_date} "+%Y%m%d.${stock}.1min.db")
+
+    SourceFileZip="${SourceFile}.gz"
+    SourceFilePath=$(${date} -d ${md_date} "+${SourceRootDir}/${StockLetter}/${stock}/%Y/${SourceFileZip}")
+    echo ${SourceFilePath}
+
+    download_file ${SourceFilePath}
+done
+
+chmod 600 ${TargetFilePath}
+ls -lh ${TargetFilePath}
+
+# mkdir -p /tmp/aaa
+# cp -r ${tmp_dir}/* /tmp/aaa/