diff --git a/release_version.txt b/release_version.txt index a0a1517..eb514eb 100644 --- a/release_version.txt +++ b/release_version.txt @@ -1 +1 @@ -0.6.3 \ No newline at end of file +0.6.4 \ No newline at end of file diff --git a/scripts/alpaca_hist_md/alpaca_md.sh b/scripts/alpaca_hist_md/alpaca_md.sh new file mode 100755 index 0000000..429377a --- /dev/null +++ b/scripts/alpaca_hist_md/alpaca_md.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +CalendarURL=http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS +is_business_day() { + dt=${1} + + open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time') + if [ -n "${open_time}" ]; then + return 0 + else + return 1 + fi + +} +export -f is_business_day + +Start=${1} +End=${2} +NumJobs=${3} + +if [ "${Start}" == "" ] ; then + echo "Usage: ${0} [] [ (25)]]" + exit +fi + +if ! is_business_day ${Start}; then + echo "${Start} is not business day in US" + exit +fi + +if [ "${End}" == "" ] ; then + End=$(date -d "${Start} + 1 day" "+%Y-%m-%d") +fi + +if [ "${NumJobs}" == "" ] ; then + NumJobs=25 +fi + +echo "Start=${Start} End=${End} NumJobs=${NumJobs}" + + +Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3 +PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_mkt_data.py + +OutputDir=/home/cvtt/alpaca_md +Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md + + +run_proc() { + Inst=${1} + Start=${2} + End=${3} + echo "Running for $Inst" + + + Cmd="${Python} ${PyScript}" + Cmd="${Cmd} --config=${Config}" + Cmd="${Cmd} --output_dir=${OutputDir}" + Cmd="${Cmd} --instruments=ALPACA:${Inst}" + Cmd="${Cmd} --start=${Start}" + if [ "${End}" != "" ]; then + Cmd="${Cmd} --end=${End}" + fi + Cmd="${Cmd} --log_file=./logs/${Start}.${Inst}.log" + echo ${Cmd} + eval ${Cmd} +} + +export PYTHONPATH=/home/cvtt/prod +export -f run_proc + +key=$(jq -r '.["ALPACA_SANDBOX"] | .api_key' ~/.creds) +secret=$(jq -r '.["ALPACA_SANDBOX"] | .secret_key' ~/.creds) + +Cmd="curl -s --request GET --url 'https://paper-api.alpaca.markets/v2/assets?status=active'" +Cmd="${Cmd} --header 'APCA-API-KEY-ID: ${key}'" +Cmd="${Cmd} --header 'APCA-API-SECRET-KEY: ${secret}'" +Cmd="${Cmd} --header 'accept: application/json'" +Cmd="${Cmd} | jq '.[] | select(.class == \"us_equity\" and .exchange != \"OTC\") | .symbol'" +Cmd="${Cmd} | sed 's/\"//g'" +Cmd="${Cmd} | sed 's/^/STOCK-/'" + +# split string into array +Instruments=() +for Inst in $(eval ${Cmd}) +do + Instruments+=("$Inst") +done + +slice_size=100 # 10K symbols parallel cannot handle +for ((ii=0; ii <${#Instruments[@]}; ii+=slice_size)); do + InstSlice=("${Instruments[@]:ii:slice_size}") + + parallel -j ${NumJobs} run_proc {} ${Start} ${End} ::: "${InstSlice[@]}" +done