diff --git a/research/load_eqty_md.sh b/research/load_eqty_md.sh new file mode 100755 index 0000000..da3f4a6 --- /dev/null +++ b/research/load_eqty_md.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash + +usage() { + echo "Usage: $0 [-h ] [-d (yesterday*)] [-s " + exit 1 +} +if [[ "$(uname)" == "Darwin" ]]; then + # macOS + date='gdate' +else + date='date' +fi + + +is_valid() { + local inst=$1 + shift + local valid_instances=("$@") + + for valid_inst in "${valid_instances[@]}"; + do + if [[ "$inst" == "$valid_inst" ]]; then + return 0 + fi + done + return 1 +} + +md_date="" +host=homestore +stocks=COIN,GBTC,SQ + +while getopts ":h:d:s:" opt; do + case ${opt} in + d ) + md_date=$OPTARG + ;; + h ) + host=$OPTARG + ;; + s ) + stocks=$OPTARG + ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + usage + ;; + : ) + echo "Option -$OPTARG requires an argument." >&2 + usage + ;; + esac +done + +if [ "${md_date}" == "" ] ; then + md_date="yesterday" +fi + +echo "$md_date $host $source" + +valid_hosts=('homestore' 'cloud21') +if ! is_valid "${host}" "${valid_hosts[@]}" ; then + echo "Host '${host}' is not valid" + usage +fi + +# ---- temp dir +echo ${SourceFile} +tmp_dir=$(mktemp -d) + +function cleanup { + cd ${HOME} + rm -rf ${tmp_dir} +} +trap cleanup EXIT +# ---- temp dir + + +if [ "${host}" == "cloud21" ] ; then + SourceHost=cloud21.cvtt.vpn + SourceUser=cvtt + SourceRootDir="/opt/store/cvtt/md_archive/equity/alpaca_md" +elif [ "${host}" == "homestore" ]; then + SourceHost=homestore.cvtt.vpn + SourceUser=oleg + SourceRootDir="/works/cvtt/md_archive/equity/alpaca_md" +else + usage +fi + + +OLDIFS=$IFS +IFS=',' +read -ra Stocks <<< "$stocks" +IFS=$OLDIFS + + +echo "${SourceHost} ${SourceUser} ${SourceRootDir} ${Stocks[@]}" + + +function download_file { + local SourceFilePath=${1} + + Cmd="rsync" + Cmd="${Cmd} -ahv" + if tty -s; then + Cmd="${Cmd} --progress=info2" + fi + Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/" + echo ${Cmd} + eval ${Cmd} + ls -lh ${tmp_dir} + Cmd="gunzip ${tmp_dir}/${SourceFileZip}" + echo ${Cmd} && eval ${Cmd} + + ls -lh ${tmp_dir} + + Cmd="sqlite3 ${tmp_dir}/${SourceFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}" + echo ${Cmd} + eval ${Cmd} + +} + +# TargetDir="/opt/jupyter_gpu/data/eqty_md" +TargetDir="/tmp/aaa" +mkdir -p ${TargetDir} + +TargetFile=$(${date} -d ${md_date} "+%Y%m%d.eqty.mktdata.ohlcv.db") +TargetFilePath="${TargetDir}/${TargetFile}" +rm -f ${TargetFilePath} +touch ${TargetFilePath} + +Table=md_1min_bars + +echo "Loading files" +for stock in "${Stocks[@]}"; do + StockLetter="${stock:0:1}" + SourceFile=$(${date} -d ${md_date} "+%Y%m%d.${stock}.1min.db") + + SourceFileZip="${SourceFile}.gz" + SourceFilePath=$(${date} -d ${md_date} "+${SourceRootDir}/${StockLetter}/${stock}/%Y/${SourceFileZip}") + echo ${SourceFilePath} + + download_file ${SourceFilePath} +done + +chmod 600 ${TargetFilePath} +ls -lh ${TargetFilePath} + +# mkdir -p /tmp/aaa +# cp -r ${tmp_dir}/* /tmp/aaa/