progress

alpaca_md_day/alpaca_md_day.sh

@@ -9,6 +9,11 @@ export PyScript=/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
 export OutputDir=/app/data/alpaca_md # Local
 export LogDir=/logs
 
+DEFAULT_NUM_JOBS=10
+SLEEP_SEC_AFTER_PROC=2
+DEFAULT_SLICE_SIZE=500
+# --- Settings
+
 mkdir -p ${OutputDir}
 mkdir -p ${LogDir}
 
@@ -61,7 +66,7 @@ if [ "${End}" == $(date '+%Y-%m-%d') ] ; then
 fi
 
 if [ "${NumJobs}" == "" ] ; then
-    NumJobs=15
+    NumJobs=${DEFAULT_NUM_JOBS}
 fi
 
 echo "Start=${Start} End=${End} NumJobs=${NumJobs}"
@@ -85,6 +90,8 @@ run_proc() {
     Cmd="${Cmd} --log_level=WARNING"
     echo ${Cmd}
     eval ${Cmd}
+
+    sleep ${SLEEP_SEC_AFTER_PROC}
 }
 
 export -f run_proc
@@ -106,7 +113,7 @@ do
     Instruments+=("$Inst")
 done
 
-slice_size=500 # 10K symbols parallel cannot handle
+slice_size=${DEFAULT_SLICE_SIZE} # 10K symbols parallel cannot handle
 for ((ii=0; ii <${#Instruments[@]}; ii+=slice_size)); do
     InstSlice=("${Instruments[@]:ii:slice_size}")