From e444f697eb62a963ddac913ff41a5fd90ddf2716 Mon Sep 17 00:00:00 2001
From: Oleg Sheynin <osheynin@gmail.com>
Date: Wed, 14 Aug 2024 20:08:12 -0400
Subject: [PATCH] progress

---
 release_version.txt      |  2 +-
 research/load_eqty_md.sh | 63 +++++++++++++++++++++++++++-------------
 2 files changed, 44 insertions(+), 21 deletions(-)

diff --git a/release_version.txt b/release_version.txt
index 092fad7..f3c587d 100644
--- a/release_version.txt
+++ b/release_version.txt
@@ -1 +1 @@
-1.4.6,removing containers for crypto_md_day
+1.4.7,adding eqty data to research
diff --git a/research/load_eqty_md.sh b/research/load_eqty_md.sh
index da3f4a6..ac44c69 100755
--- a/research/load_eqty_md.sh
+++ b/research/load_eqty_md.sh
@@ -1,7 +1,12 @@
 #!/usr/bin/env bash
 
 usage() {
-  echo "Usage: $0 [-h <host (homestore*/cloud21)>] [-d <YYYYMMDD> (yesterday*)] [-s <stocks comma separated>"
+  echo -n "Usage: $0"
+  echo -n " [-h <host (homestore*/cloud21)>]"
+  echo -n " [-d <YYYYMMDD> (yesterday*)]"
+  echo -n " [-s <stocks comma separated>"
+  echo -n " [-t <target directory>"
+  echo
   exit 1
 }
 if [[ "$(uname)" == "Darwin" ]]; then
@@ -11,6 +16,18 @@ else
     date='date'
 fi
 
+# ------------------ Settings
+md_date=""
+host=homestore
+stocks=COIN,GBTC,SQ
+
+TargetDir="/opt/jupyter_gpu/data/eqty_md"
+mkdir -p ${TargetDir}
+
+Table=md_1min_bars
+# ------------------ Settings
+
+
 
 is_valid() {
   local inst=$1
@@ -26,11 +43,8 @@ is_valid() {
   return 1
 }
 
-md_date=""
-host=homestore
-stocks=COIN,GBTC,SQ
 
-while getopts ":h:d:s:" opt; do
+while getopts ":h:d:s:t:u" opt; do
   case ${opt} in
     d )
       md_date=$OPTARG
@@ -41,6 +55,12 @@ while getopts ":h:d:s:" opt; do
     s )
       stocks=$OPTARG
       ;;
+    t )
+      TargetDir=$OPTARG
+      ;;
+    u )
+      usage
+      ;;
     \? )
       echo "Invalid option: -$OPTARG" >&2
       usage
@@ -64,6 +84,13 @@ if ! is_valid "${host}"  "${valid_hosts[@]}" ; then
     usage
 fi
 
+TargetFile=$(${date} -d ${md_date} "+%Y%m%d.eqty.mktdata.ohlcv.db")
+
+TargetFilePath="${TargetDir}/${TargetFile}"
+mv ${TargetFilePath} "${TargetFilePath}.saved.$(${date} '+%Y%m%d_%H%M%S')"
+touch ${TargetFilePath}
+
+
 # ---- temp dir
 echo ${SourceFile}
 tmp_dir=$(mktemp -d)
@@ -112,26 +139,22 @@ function download_file {
     ls -lh ${tmp_dir}
     Cmd="gunzip ${tmp_dir}/${SourceFileZip}"
     echo ${Cmd} && eval ${Cmd}
-    
-    ls -lh ${tmp_dir}
-    
-    Cmd="sqlite3 ${tmp_dir}/${SourceFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}"
+
+    DbFile=${tmp_dir}/${SourceFile}
+
+# --- T E M P - for files older than 20240813
+    sqlite3 ${DbFile} <<EOF
+DROP INDEX md_1min_bars_idx;
+CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_idx on md_1min_bars(tstamp, exchange_id, instrument_id);
+EOF
+# --- T E M P
+
+    Cmd="sqlite3 ${DbFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}"
     echo ${Cmd}
     eval ${Cmd}
 
 }
 
-# TargetDir="/opt/jupyter_gpu/data/eqty_md"
-TargetDir="/tmp/aaa"
-mkdir -p ${TargetDir}
-
-TargetFile=$(${date} -d ${md_date} "+%Y%m%d.eqty.mktdata.ohlcv.db")
-TargetFilePath="${TargetDir}/${TargetFile}"
-rm -f ${TargetFilePath}
-touch ${TargetFilePath}
-
-Table=md_1min_bars
-
 echo "Loading files"
 for stock in "${Stocks[@]}"; do
     StockLetter="${stock:0:1}"