archive/ops
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improvements to alpaca md

Browse Source
This commit is contained in:
Oleg Sheynin 2024-03-26 11:30:41 -04:00
parent 8607db6506
commit 50ba171e32
3 changed files with 56 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
release_version.txt
View File

@ -1 +1 @@
0.6.7 0.6.8

16
scripts/alpaca_hist_md/alpaca_md.sh
View File

@ -1,9 +1,8 @@
#!/bin/bash #!/bin/bash
Start=${1} Start=${1}
End=${2} NumJobs=${2}
NumJobs=${3} InstListFile=${3}
InstListFile=${4}
export CalendarURL=http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS export CalendarURL=http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
@ -30,12 +29,10 @@ if ! is_business_day ${Start}; then
exit exit
fi fi
if [ "${End}" == "" ] ; then End=$(date -d "${Start} + 1 day" "+%Y-%m-%d")
End=$(date -d "${Start} + 1 day" "+%Y-%m-%d")
fi
if [ "${NumJobs}" == "" ] ; then if [ "${NumJobs}" == "" ] ; then
NumJobs=25 NumJobs=30
fi fi
echo "Start=${Start} End=${End} NumJobs=${NumJobs}" echo "Start=${Start} End=${End} NumJobs=${NumJobs}"
@ -44,8 +41,7 @@ export PYTHONPATH=/home/cvtt/prod
export Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3 export Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3
export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_mkt_data.py export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_mkt_data.py
# export OutputDir=/home/cvtt/alpaca_md export OutputDir=/home/cvtt/host_drive/alpaca_md # Local
export OutputDir=/home/cvtt/host_drive/alpaca_md
export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
export LogDir=/home/cvtt/prod/logs/alpaca_md export LogDir=/home/cvtt/prod/logs/alpaca_md
@ -95,7 +91,7 @@ do
Instruments+=("$Inst") Instruments+=("$Inst")
done done
slice_size=100 # 10K symbols parallel cannot handle slice_size=500 # 10K symbols parallel cannot handle
for ((ii=0; ii <${#Instruments[@]}; ii+=slice_size)); do for ((ii=0; ii <${#Instruments[@]}; ii+=slice_size)); do
InstSlice=("${Instruments[@]:ii:slice_size}") InstSlice=("${Instruments[@]:ii:slice_size}")

49
scripts/alpaca_hist_md/run_alpaca_hist_md.sh Executable file
View File

@ -0,0 +1,49 @@
#!/bin/bash
is_business_day() {
dt=${1}
open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
if [ -n "${open_time}" ]; then
return 0
else
return 1
fi
}
export -f is_business_day
# Dates are in descending order
FinishedDateFile=${HOME}/prod/run/alpaca_md_last_date
Script=${HOME}/prod/run/alpaca_md.sh
last_date=$(cat ${FinishedDateFile})
if [ "${last_date}" == "" ]; then
echo "process is currently on"
exit
fi
while true; do
new_date=$(date -d "${last_date} - 1 day" "+%Y-%m-%d")
if is_business_day ${new_date}; then
break
else
last_date=${new_date}
fi
done
echo Collecting data for ${new_date}
Cmd="${Script} ${new_date}"
echo "${0} Starting at $(date)"
echo > ${FinishedDateFile}
echo ${Cmd} && eval ${Cmd}
echo $new_date > ${FinishedDateFile}
echo "${0} Done at $(date)"