archive/ops
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

eqt initial

Browse Source
This commit is contained in:
Oleg Sheynin 2025-05-18 20:46:58 -04:00
parent 19c7313326
commit a7144a52e2
3 changed files with 39 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
VERSION
View File

@ -1 +1 @@
2.1.8,docker prepare sim - crypto - progress 2.2.1,docker prepare sim - eqt

4
scripts/mkt_data/sim_prepare/prepare_crypto_simdata.sh
View File

@ -27,7 +27,7 @@ SourceDir="${SOURCE_ROOT_DIR}/${DB_SOURCE}/${year}/${month}"
SourceFile="${DATE}.mktdata.db.gz" SourceFile="${DATE}.mktdata.db.gz"
SourceFilePath="${SourceDir}/${SourceFile}" SourceFilePath="${SourceDir}/${SourceFile}"
Cmd="rsync -ahv" Cmd="/usr/bin/rsync -ahv"
Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'" Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
Cmd+=" ${SOURCE_HOST}:${SourceFilePath}" Cmd+=" ${SOURCE_HOST}:${SourceFilePath}"
Cmd+=" $OUTPUT_DIR/" Cmd+=" $OUTPUT_DIR/"
@ -232,7 +232,7 @@ echo ${Cmd}
eval ${Cmd} eval ${Cmd}
for tgt in ${RSYNC_TARGETS} ; do for tgt in ${RSYNC_TARGETS} ; do
Cmd="rsync -ahv" Cmd="/usr/bin/rsync -ahv"
Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'" Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
Cmd+=" ${ResultDbFile}.gz ${tgt}" Cmd+=" ${ResultDbFile}.gz ${tgt}"
echo ${Cmd} echo ${Cmd}

102
scripts/mkt_data/sim_prepare/prepare_eqt_simdata.sh
View File

@ -1,13 +1,15 @@
#!/bin/bash #!/bin/bash
usage() {
echo "Usage: $0 -S <symbols> -d <YYYYMMDD Date> [-O <output dir (./) >]"
exit 1
}
# --------------------- Settings # --------------------- Settings
SourceHost=cloud21.cvtt.vpn SOURCE_HOST=cloud21.cvtt.vpn
SourceRootDir=/opt/store/cvtt/md_archive/equity/alpaca_md SOURCE_ROOT_DIR=/opt/store/cvtt/md_archive/equity/alpaca_md
SYMBOL_LIST=CAN #,COIN,GBTC,HOOD,MSTR,PYPL,XYZ
OUTPUT_DIR=/tmp
DATE=20250514
RSYNC_TARGETS="cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/test/alpaca_md/sim/"
# RSYNC_TARGETS="cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/ cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"
# --------------------- Settings
# --------------------- Settings # --------------------- Settings
@ -27,79 +29,51 @@ is_business_day() {
} }
export -f is_business_day export -f is_business_day
if [ -z ${SYMBOL_LIST} ] ; then
while getopts ":d:S:O:" opt; do
case ${opt} in
d )
Date=$OPTARG
;;
S )
SymList=$OPTARG
;;
O )
OutputDir=$OPTARG
;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
if [ -z ${SymList} ] ; then
echo "symbols are not specified" echo "symbols are not specified"
usage exit
fi fi
if [ -z ${Date} ] ; then if [ -z ${DATE} ] ; then
echo "date is not specified" DATE=$(date -d 'yesterday' +'%Y%m%d')
usage
fi fi
if [ -z ${OutputDir} ] ; then
OutputDir=.
fi
mkdir -p ${OutputDir}
if ! is_business_day ${Date}; then mkdir -p ${OUTPUT_DIR}
echo "${Date} is not business day"
if ! is_business_day ${DATE}; then
echo "${DATE} is not business day"
usage usage
fi fi
OLD_IFS=${IFS} OLD_IFS=${IFS}
IFS="," IFS=","
read -ra Symbols <<< "${SymList}" read -ra Symbols <<< "${SYMBOL_LIST}"
IFS=${OLD_IFS} IFS=${OLD_IFS}
echo "Date=${Date} Symbols=${Symbols[@]} OutputDir=${OutputDir}" echo "DATE=${DATE} Symbols=${Symbols[@]} OUTPUT_DIR=${OUTPUT_DIR}"
echo Getting data from ${DataHost} ... echo Getting data from ${DataHost} ...
year=$(date -d ${Date} +"%Y") year=$(date -d ${DATE} +"%Y")
for sym in ${Symbols[@]}; do for sym in ${Symbols[@]}; do
inst_id="STOCK-${sym}" inst_id="STOCK-${sym}"
capital=${sym:0:1} capital=${sym:0:1}
SourceDir="${SourceRootDir}/${year}/${capital}/${sym}" SourceDir="${SOURCE_ROOT_DIR}/${year}/${capital}/${sym}"
SourceHbarFile="${SourceDir}/${Date}.${sym}.alpaca_1m_bars.db.gz" SourceHbarFile="${SourceDir}/${DATE}.${sym}.alpaca_1m_bars.db.gz"
SourceQatFile="${SourceDir}/${Date}.${sym}.alpaca_qat.db.gz" SourceQatFile="${SourceDir}/${DATE}.${sym}.alpaca_qat.db.gz"
for src_file in ${SourceHbarFile} ${SourceQatFile}; do for src_file in ${SourceHbarFile} ${SourceQatFile}; do
Cmd="rsync -ahv" Cmd="rsync -ahv"
Cmd+=" ${SourceHost}:${src_file}" Cmd+=" ${SOURCE_HOST}:${src_file}"
Cmd+=" $OutputDir/" Cmd+=" $OUTPUT_DIR/"
echo ${Cmd} echo ${Cmd}
eval ${Cmd} eval ${Cmd}
done done
done done
Cmd="(cd ${OutputDir} && gunzip *.db.gz)" Cmd="(cd ${OUTPUT_DIR} && gunzip *.db.gz)"
echo ${Cmd} echo ${Cmd}
eval ${Cmd} eval ${Cmd}
ResultDbFile="${OutputDir}/${Date}.alpaca_sim_md.db" ResultDbFile="${OUTPUT_DIR}/${DATE}.alpaca_sim_md.db"
echo "Creating Result Database File ${ResultDbFile}"
echo "Creating Result Database File ${ResultDbFile}" echo "Creating Result Database File ${ResultDbFile}"
echo "Creating table md_trades ..." echo "Creating table md_trades ..."
@ -163,8 +137,8 @@ EOF
# set -f # not to expand * # set -f # not to expand *
for sym in ${Symbols[@]}; do for sym in ${Symbols[@]}; do
src_hbar_db=${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db src_hbar_db=${OUTPUT_DIR}/${DATE}.${sym}.alpaca_1m_bars.db
src_qat_db=${OutputDir}/${Date}.${sym}.alpaca_qat.db src_qat_db=${OUTPUT_DIR}/${DATE}.${sym}.alpaca_qat.db
echo "Loading md_trades and md_quotes from ${src_qat_db} ..." echo "Loading md_trades and md_quotes from ${src_qat_db} ..."
sqlite3 ${ResultDbFile} <<EOF sqlite3 ${ResultDbFile} <<EOF
@ -233,16 +207,12 @@ Cmd="gzip ${ResultDbFile}"
echo ${Cmd} echo ${Cmd}
eval ${Cmd} eval ${Cmd}
Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/" for tgt in ${RSYNC_TARGETS} ; do
echo ${Cmd} Cmd="/usr/bin/rsync -ahv"
eval ${Cmd} Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
Cmd+=" ${ResultDbFile}.gz ${tgt}"
Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/" echo ${Cmd}
echo ${Cmd} eval ${Cmd}
eval ${Cmd} done
Cmd="rm ${ResultDbFile}.gz"
echo ${Cmd}
eval ${Cmd}
echo Done $0 ${*} echo Done $0 ${*}