152 lines
3.3 KiB
Bash
Executable File
152 lines
3.3 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
usage() {
|
|
echo "Usage: $0 -S <symbols> -d <YYYYMMDD Date> [-O <output dir (./) >]"
|
|
exit 1
|
|
}
|
|
|
|
# --------------------- Settings
|
|
SourceHost=cloud21.cvtt.vpn
|
|
SourceRootDir=/opt/store/cvtt/md_archive/equity/alpaca_md
|
|
# --------------------- Settings
|
|
|
|
|
|
is_business_day() {
|
|
dt=${1}
|
|
date=$(date -d "${dt}" +"%Y-%m-%d")
|
|
|
|
CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
|
|
URL="${CalendarURL}&start=${date}&end=${date}"
|
|
open_time=$(curl -s "${URL}" | jq '.[] | .open_time')
|
|
if [ -n "${open_time}" ]; then
|
|
return 0
|
|
else
|
|
return 1
|
|
fi
|
|
|
|
}
|
|
export -f is_business_day
|
|
|
|
|
|
while getopts ":d:S:O:" opt; do
|
|
case ${opt} in
|
|
d )
|
|
Date=$OPTARG
|
|
;;
|
|
S )
|
|
SymList=$OPTARG
|
|
;;
|
|
O )
|
|
OutputDir=$OPTARG
|
|
;;
|
|
\? )
|
|
echo "Invalid option: -$OPTARG" >&2
|
|
usage
|
|
;;
|
|
: )
|
|
echo "Option -$OPTARG requires an argument." >&2
|
|
usage
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [ -z ${SymList} ] ; then
|
|
echo "symbols are not specified"
|
|
usage
|
|
fi
|
|
if [ -z ${Date} ] ; then
|
|
echo "date is not specified"
|
|
usage
|
|
fi
|
|
if [ -z ${OutputDir} ] ; then
|
|
OutputDir=.
|
|
fi
|
|
mkdir -p ${OutputDir}
|
|
|
|
if ! is_business_day ${Date}; then
|
|
echo "${Date} is not business day"
|
|
usage
|
|
fi
|
|
|
|
OLD_IFS=${IFS}
|
|
IFS=","
|
|
read -ra Symbols <<< "${SymList}"
|
|
IFS=${OLD_IFS}
|
|
|
|
echo "Date=${Date} Symbols=${Symbols[@]} OutputDir=${OutputDir}"
|
|
echo Getting data from ${DataHost} ...
|
|
|
|
year=$(date -d ${Date} +"%Y")
|
|
for sym in ${Symbols[@]}; do
|
|
inst_id="STOCK-${sym}"
|
|
capital=${sym:0:1}
|
|
SourceDir="${SourceRootDir}/${year}/${capital}/${sym}"
|
|
SourceHbarFile="${SourceDir}/${Date}.${sym}.alpaca_1m_bars.db.gz"
|
|
SourceQatFile="${SourceDir}/${Date}.${sym}.alpaca_qat.db.gz"
|
|
|
|
for src_file in ${SourceHbarFile} ${SourceQatFile}; do
|
|
Cmd="rsync -ahv"
|
|
Cmd+=" ${SourceHost}:${src_file}"
|
|
Cmd+=" $OutputDir/"
|
|
echo ${Cmd}
|
|
eval ${Cmd}
|
|
done
|
|
done
|
|
|
|
Cmd="(cd ${OutputDir} && gunzip *.db.gz)"
|
|
echo ${Cmd}
|
|
eval ${Cmd}
|
|
|
|
ResultDbFile="${OutputDir}/${Date}.alpaca_sim_md.db"
|
|
echo "Creating Result Database File ${ResultDbFile}"
|
|
sym=${Symbols[0]}
|
|
for db_file in ${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db ${OutputDir}/${Date}.${sym}.alpaca_qat.db ; do
|
|
Cmd="sqlite3"
|
|
Cmd+=" ${db_file} .schema"
|
|
Cmd+=" | sqlite3 ${ResultDbFile}"
|
|
echo ${Cmd}
|
|
eval ${Cmd}
|
|
done
|
|
|
|
set -f # not to expand *
|
|
|
|
for sym in ${Symbols[@]}; do
|
|
src_hbar_db=${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db
|
|
src_qat_db=${OutputDir}/${Date}.${sym}.alpaca_qat.db
|
|
for srcdb in ${src_hbar_db} ${src_qat_db} ; do
|
|
|
|
tables=$(sqlite3 ${srcdb} .tables)
|
|
for table in $(sqlite3 ${srcdb} .tables) ; do
|
|
echo "Loading from ${srcdb}.${table} ..."
|
|
sqlite3 ${ResultDbFile} <<EOF
|
|
ATTACH '${srcdb}' AS source_db;
|
|
BEGIN;
|
|
INSERT INTO ${table} SELECT * FROM source_db.${table};
|
|
COMMIT;
|
|
DETACH source_db;
|
|
EOF
|
|
done
|
|
Cmd="rm ${srcdb}"
|
|
echo ${Cmd}
|
|
eval ${Cmd}
|
|
done
|
|
done
|
|
|
|
Cmd="gzip ${ResultDbFile}"
|
|
echo ${Cmd}
|
|
eval ${Cmd}
|
|
|
|
Cmd="rsync -ahvv ${ResultDbFile}.gz homestore.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/"
|
|
echo ${Cmd}
|
|
eval ${Cmd}
|
|
|
|
Cmd="rsync -ahvv ${ResultDbFile}.gz cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"
|
|
echo ${Cmd}
|
|
eval ${Cmd}
|
|
|
|
Cmd="rm ${ResultDbFile}.gz"
|
|
echo ${Cmd}
|
|
eval ${Cmd}
|
|
|
|
echo Done $0 ${*}
|