diff --git a/release_version.txt b/release_version.txt index ac055ee..e6edfcd 100644 --- a/release_version.txt +++ b/release_version.txt @@ -1 +1 @@ -1.8.9,prepare sim +1.9.1,16->23 diff --git a/research/prepare_crypto_simdata.sh b/research/prepare_crypto_simdata.sh new file mode 100755 index 0000000..1c2a014 --- /dev/null +++ b/research/prepare_crypto_simdata.sh @@ -0,0 +1,151 @@ +#!/bin/bash + +usage() { + echo "Usage: $0 -S -d [-O ]" + exit 1 +} + +# --------------------- Settings +SourceHost=cloud21.cvtt.vpn +SourceRootDir=/opt/store/cvtt/md_archive/crypto/alpaca_md +# --------------------- Settings + + +is_business_day() { + dt=${1} + date=$(date -d "${dt}" +"%Y-%m-%d") + + CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS + URL="${CalendarURL}&start=${date}&end=${date}" + open_time=$(curl -s "${URL}" | jq '.[] | .open_time') + if [ -n "${open_time}" ]; then + return 0 + else + return 1 + fi + +} +export -f is_business_day + + +while getopts ":d:S:O:" opt; do + case ${opt} in + d ) + Date=$OPTARG + ;; + S ) + SymList=$OPTARG + ;; + O ) + OutputDir=$OPTARG + ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + usage + ;; + : ) + echo "Option -$OPTARG requires an argument." >&2 + usage + ;; + esac +done + +if [ -z ${SymList} ] ; then + echo "symbols are not specified" + usage +fi +if [ -z ${Date} ] ; then + echo "date is not specified" + usage +fi +if [ -z ${OutputDir} ] ; then + OutputDir=. +fi +mkdir -p ${OutputDir} + +if ! is_business_day ${Date}; then + echo "${Date} is not business day" + usage +fi + +OLD_IFS=${IFS} +IFS="," +read -ra Symbols <<< "${SymList}" +IFS=${OLD_IFS} + +echo "Date=${Date} Symbols=${Symbols[@]} OutputDir=${OutputDir}" +echo Getting data from ${DataHost} ... + +year=$(date -d ${Date} +"%Y") +for sym in ${Symbols[@]}; do + inst_id="STOCK-${sym}" + capital=${sym:0:1} + SourceDir="${SourceRootDir}/${year}/${capital}/${sym}" + SourceHbarFile="${SourceDir}/${Date}.${sym}.alpaca_1m_bars.db.gz" + SourceQatFile="${SourceDir}/${Date}.${sym}.alpaca_qat.db.gz" + + for src_file in ${SourceHbarFile} ${SourceQatFile}; do + Cmd="rsync -ahv" + Cmd+=" ${SourceHost}:${src_file}" + Cmd+=" $OutputDir/" + echo ${Cmd} + eval ${Cmd} + done +done + +Cmd="(cd ${OutputDir} && gunzip *.db.gz)" +echo ${Cmd} +eval ${Cmd} + +ResultDbFile="${OutputDir}/${Date}.alpaca_sim_md.db" +echo "Creating Result Database File ${ResultDbFile}" +sym=${Symbols[0]} +for db_file in ${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db ${OutputDir}/${Date}.${sym}.alpaca_qat.db ; do + Cmd="sqlite3" + Cmd+=" ${db_file} .schema" + Cmd+=" | sqlite3 ${ResultDbFile}" + echo ${Cmd} + eval ${Cmd} +done + +set -f # not to expand * + +for sym in ${Symbols[@]}; do + src_hbar_db=${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db + src_qat_db=${OutputDir}/${Date}.${sym}.alpaca_qat.db + for srcdb in ${src_hbar_db} ${src_qat_db} ; do + + tables=$(sqlite3 ${srcdb} .tables) + for table in $(sqlite3 ${srcdb} .tables) ; do + echo "Loading from ${srcdb}.${table} ..." + sqlite3 ${ResultDbFile} <