From df761027eca836864ade491f056de5a2f841d78e Mon Sep 17 00:00:00 2001
From: Oleg Sheynin <osheynin@gmail.com>
Date: Mon, 5 Aug 2024 22:41:52 -0400
Subject: [PATCH] progress

---
 release_version.txt                         |   2 +-
 research/{ => legacy}/load_histdata.sh      |   0
 research/{ => legacy}/prune_eqt_histdata.sh |   0
 research/load_crypto_md.sh                  | 126 ++++++++++++++++++++
 4 files changed, 127 insertions(+), 1 deletion(-)
 rename research/{ => legacy}/load_histdata.sh (100%)
 rename research/{ => legacy}/prune_eqt_histdata.sh (100%)
 create mode 100755 research/load_crypto_md.sh

diff --git a/release_version.txt b/release_version.txt
index 31e5c84..d0149fe 100644
--- a/release_version.txt
+++ b/release_version.txt
@@ -1 +1 @@
-1.3.3
+1.3.4
diff --git a/research/load_histdata.sh b/research/legacy/load_histdata.sh
similarity index 100%
rename from research/load_histdata.sh
rename to research/legacy/load_histdata.sh
diff --git a/research/prune_eqt_histdata.sh b/research/legacy/prune_eqt_histdata.sh
similarity index 100%
rename from research/prune_eqt_histdata.sh
rename to research/legacy/prune_eqt_histdata.sh
diff --git a/research/load_crypto_md.sh b/research/load_crypto_md.sh
new file mode 100755
index 0000000..4ed7d3c
--- /dev/null
+++ b/research/load_crypto_md.sh
@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+
+usage() {
+  echo "Usage: $0 [-h <host (homestore*/cloud21)>] [-d <YYYYMMDD> (yesterday*)] [-s <source> (cvttdata/cloud21*)>]"
+  exit 1
+}
+
+is_valid() {
+  local inst=$1
+  local valid_instances=$2
+
+  for valid_inst in "${valid_instances[@]}";
+  do
+    if [[ "$inst" == "$valid_inst" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+date=""
+host=homestore
+source=cloud21
+
+while getopts ":h:d:s:" opt; do
+  case ${opt} in
+    d )
+      date=$OPTARG
+      ;;
+    h )
+      host=$OPTARG
+      ;;
+    s )
+      source=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+if [ "${date}" == "" ] ; then
+    date="yesterday"
+fi
+
+echo "$date $host $source"
+
+valid_hosts=('homestore' 'cloud21')
+if ! is_valid "${host}"  "${valid_hosts[@]}" ; then
+    echo h
+    usage
+fi
+
+valid_sources=('cloud21' 'cvttdata')
+if ! is_valid ${source} ${valid_sources[@]} ; then
+    echo s
+    usage
+fi
+
+
+if [ "${host}" == "cloud21" ] ; then
+    SourceHost=cloud21.cvtt.vpn
+    SourceUser=cvtt
+    SourceRootDir="/opt/store/cvtt/md_archive/crypto/${source}"
+elif [ "${host}" == "homestore" ]; then
+    SourceHost=homestore.cvtt.vpn
+    SourceUser=oleg
+    SourceRootDir=/works/cvtt/md_archive/crypto/${source}
+else
+    usage
+fi
+
+echo "${SourceHost} ${SourceUser} ${SourceRootDir}"
+
+SourceFile=$(date -d ${date} "+%Y%m%d.mktdata.db")
+SourceFileZip="${SourceFile}.gz"
+SourceFilePath=$(date -d ${date} "+${SourceRootDir}/%Y/%m/${SourceFileZip}")
+
+TargetDir="/opt/jupyter_gpu/data/crypto_md"
+TargetFile=$(date -d ${date} "+%Y%m%d.mktdata.ohlcv.db")
+TargetFilePath="${TargetDir}/${TargetFile}"
+
+echo ${SourceFile}
+tmp_dir=$(mktemp -d)
+
+function cleanup {
+    cd ${HOME}
+    rm -rf ${tmp_dir}
+}
+trap cleanup EXIT
+
+function download_file {
+
+    Cmd="rsync"
+    Cmd="${Cmd} -ahv"
+    if tty -s; then
+        Cmd="${Cmd} --progress=info2"
+    fi
+    Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/"
+    echo ${Cmd}
+    eval ${Cmd}
+    ls -lh ${tmp_dir}
+    Cmd="gunzip ${tmp_dir}/${SourceFileZip}"
+    echo ${Cmd} && eval ${Cmd}
+    ls -lh ${tmp_dir}
+
+    rm -f ${TargetFilePath}
+    touch ${TargetFilePath}
+
+    for table in bnbfut_ohlcv_1min bnbspot_ohlcv_1min coinbase_ohlcv_1min
+    do
+        Cmd="sqlite3 ${tmp_dir}/${SourceFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}"
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+    chmod 600 ${TargetFilePath}
+    ls -lh ${TargetFilePath}
+
+
+}
+download_file