diff --git a/VERSION b/VERSION index 13344bd..43eabee 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1.8,docker prepare sim - crypto - progress +2.2.1,docker prepare sim - eqt diff --git a/scripts/mkt_data/sim_prepare/prepare_crypto_simdata.sh b/scripts/mkt_data/sim_prepare/prepare_crypto_simdata.sh index 8caadb8..94f3983 100755 --- a/scripts/mkt_data/sim_prepare/prepare_crypto_simdata.sh +++ b/scripts/mkt_data/sim_prepare/prepare_crypto_simdata.sh @@ -27,7 +27,7 @@ SourceDir="${SOURCE_ROOT_DIR}/${DB_SOURCE}/${year}/${month}" SourceFile="${DATE}.mktdata.db.gz" SourceFilePath="${SourceDir}/${SourceFile}" -Cmd="rsync -ahv" +Cmd="/usr/bin/rsync -ahv" Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'" Cmd+=" ${SOURCE_HOST}:${SourceFilePath}" Cmd+=" $OUTPUT_DIR/" @@ -232,7 +232,7 @@ echo ${Cmd} eval ${Cmd} for tgt in ${RSYNC_TARGETS} ; do - Cmd="rsync -ahv" + Cmd="/usr/bin/rsync -ahv" Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'" Cmd+=" ${ResultDbFile}.gz ${tgt}" echo ${Cmd} diff --git a/scripts/mkt_data/sim_prepare/prepare_eqt_simdata.sh b/scripts/mkt_data/sim_prepare/prepare_eqt_simdata.sh index 19dcd61..8e0a6a9 100755 --- a/scripts/mkt_data/sim_prepare/prepare_eqt_simdata.sh +++ b/scripts/mkt_data/sim_prepare/prepare_eqt_simdata.sh @@ -1,13 +1,15 @@ #!/bin/bash -usage() { - echo "Usage: $0 -S -d [-O ]" - exit 1 -} - # --------------------- Settings -SourceHost=cloud21.cvtt.vpn -SourceRootDir=/opt/store/cvtt/md_archive/equity/alpaca_md +SOURCE_HOST=cloud21.cvtt.vpn +SOURCE_ROOT_DIR=/opt/store/cvtt/md_archive/equity/alpaca_md +SYMBOL_LIST=CAN #,COIN,GBTC,HOOD,MSTR,PYPL,XYZ +OUTPUT_DIR=/tmp +DATE=20250514 +RSYNC_TARGETS="cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/test/alpaca_md/sim/" +# RSYNC_TARGETS="cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/ cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/" +# --------------------- Settings + # --------------------- Settings @@ -27,79 +29,51 @@ is_business_day() { } export -f is_business_day - -while getopts ":d:S:O:" opt; do - case ${opt} in - d ) - Date=$OPTARG - ;; - S ) - SymList=$OPTARG - ;; - O ) - OutputDir=$OPTARG - ;; - \? ) - echo "Invalid option: -$OPTARG" >&2 - usage - ;; - : ) - echo "Option -$OPTARG requires an argument." >&2 - usage - ;; - esac -done - -if [ -z ${SymList} ] ; then +if [ -z ${SYMBOL_LIST} ] ; then echo "symbols are not specified" - usage + exit fi -if [ -z ${Date} ] ; then - echo "date is not specified" - usage +if [ -z ${DATE} ] ; then + DATE=$(date -d 'yesterday' +'%Y%m%d') fi -if [ -z ${OutputDir} ] ; then - OutputDir=. -fi -mkdir -p ${OutputDir} -if ! is_business_day ${Date}; then - echo "${Date} is not business day" +mkdir -p ${OUTPUT_DIR} + +if ! is_business_day ${DATE}; then + echo "${DATE} is not business day" usage fi OLD_IFS=${IFS} IFS="," -read -ra Symbols <<< "${SymList}" +read -ra Symbols <<< "${SYMBOL_LIST}" IFS=${OLD_IFS} -echo "Date=${Date} Symbols=${Symbols[@]} OutputDir=${OutputDir}" +echo "DATE=${DATE} Symbols=${Symbols[@]} OUTPUT_DIR=${OUTPUT_DIR}" echo Getting data from ${DataHost} ... -year=$(date -d ${Date} +"%Y") +year=$(date -d ${DATE} +"%Y") for sym in ${Symbols[@]}; do inst_id="STOCK-${sym}" capital=${sym:0:1} - SourceDir="${SourceRootDir}/${year}/${capital}/${sym}" - SourceHbarFile="${SourceDir}/${Date}.${sym}.alpaca_1m_bars.db.gz" - SourceQatFile="${SourceDir}/${Date}.${sym}.alpaca_qat.db.gz" + SourceDir="${SOURCE_ROOT_DIR}/${year}/${capital}/${sym}" + SourceHbarFile="${SourceDir}/${DATE}.${sym}.alpaca_1m_bars.db.gz" + SourceQatFile="${SourceDir}/${DATE}.${sym}.alpaca_qat.db.gz" for src_file in ${SourceHbarFile} ${SourceQatFile}; do Cmd="rsync -ahv" - Cmd+=" ${SourceHost}:${src_file}" - Cmd+=" $OutputDir/" + Cmd+=" ${SOURCE_HOST}:${src_file}" + Cmd+=" $OUTPUT_DIR/" echo ${Cmd} eval ${Cmd} done done -Cmd="(cd ${OutputDir} && gunzip *.db.gz)" +Cmd="(cd ${OUTPUT_DIR} && gunzip *.db.gz)" echo ${Cmd} eval ${Cmd} -ResultDbFile="${OutputDir}/${Date}.alpaca_sim_md.db" -echo "Creating Result Database File ${ResultDbFile}" - +ResultDbFile="${OUTPUT_DIR}/${DATE}.alpaca_sim_md.db" echo "Creating Result Database File ${ResultDbFile}" echo "Creating table md_trades ..." @@ -163,8 +137,8 @@ EOF # set -f # not to expand * for sym in ${Symbols[@]}; do - src_hbar_db=${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db - src_qat_db=${OutputDir}/${Date}.${sym}.alpaca_qat.db + src_hbar_db=${OUTPUT_DIR}/${DATE}.${sym}.alpaca_1m_bars.db + src_qat_db=${OUTPUT_DIR}/${DATE}.${sym}.alpaca_qat.db echo "Loading md_trades and md_quotes from ${src_qat_db} ..." sqlite3 ${ResultDbFile} <