load eqt to research - fix

trading recorder -> docker
2024-08-28 10:49:07 -04:00 · 2024-08-26 16:59:01 -04:00 · 2024-08-24 19:19:18 -04:00 · 2024-08-24 18:23:27 -04:00 · 2024-08-22 18:36:29 -04:00 · 2024-08-22 18:31:14 -04:00
8 changed files with 425 additions and 13 deletions
@@ -1,15 +1,30 @@
 #!/bin/bash

+usage() {
+  echo -n "Usage: ${0}"
+  echo -n " [-C <config (dflt: apps/cvtt_eqt_alpaca)>]"
+  echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
+  echo -n " [-e <active_exchanges (ALPACA_SNBOX)>]"
+  echo -n " [-a <admin_port (7220)>]"
+  echo -n " [-n <portal_name (MD_PORTAL_ALPACA)>]"
+  echo
+  exit 1
+}
+
+
 args=${*}

+Config=apps/cvtt_eqt_alpaca
 ConfigServer=cloud23.cvtt.vpn:6789
 ActiveExchanges=ALPACA_SNDBX
-PortalName=MD_PORTAL_ALPACA
+PortalName=MD_PORTAL_ALPACA_SNDBX
 AdminPort=7220

-
-while getopts ":c:e:a:n:" opt; do
+while getopts ":C:c:e:a:n:" opt; do
  case ${opt} in
+    C )
+      Config=$OPTARG
+      ;;
    c )
      ConfigServer=$OPTARG
      ;;
@@ -33,6 +48,7 @@ while getopts ":c:e:a:n:" opt; do
  esac
 done

+echo "Config=${Config}"
 echo "ConfigServer=${ConfigServer}"
 echo "ActiveExchanges=${ActiveExchanges}"
 echo "PortalName=${PortalName}"
@@ -43,10 +59,6 @@ ImageName=md_portal
 DockerRegistry=cloud21.cvtt.vpn:5500
 DockerImage=${DockerRegistry}/${ImageName}

-if [ -z ${date} ] ; then
-    date=$(date -d "yesterday" +%Y%m%d)
-fi
-
 Cmd="docker run"
 Cmd="${Cmd} --detach"
 Cmd="${Cmd} --restart=unless-stopped"
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+usage() {
+  echo -n "Usage: $0"
+  echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
+  echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+  echo -n " [-a <admin_port (7222)>"]
+  echo
+  exit 1
+}
+
+args=${*}
+
+ConfigServer=cloud23.cvtt.vpn:6789
+ActiveExchanges=ALPACA_SNDBX-MDPORTAL
+AdminPort=7222
+Book=""
+
+while getopts ":c:e:a:" opt; do
+  case ${opt} in
+    c )
+      ConfigServer=$OPTARG
+      ;;
+    e )
+      ActiveExchanges=$OPTARG
+      ;;
+    a )
+      AdminPort=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+echo "ConfigServer=${ConfigServer}"
+echo "ActiveExchanges=${ActiveExchanges}"
+echo "AdminPort=${AdminPort}"
+
+
+ImageName=executor
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/${ImageName}
+
+Cmd="docker run"
+Cmd="${Cmd} --detach"
+Cmd="${Cmd} --restart=unless-stopped"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ImageName}"
+Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${args}"
+
+echo $Cmd
+eval $Cmd
@@ -0,0 +1,86 @@
+#!/bin/bash
+
+usage() {
+  echo -n "Usage: $0"
+  echo -n " -b <book>"
+
+  echo -n " -S <strategy> (dflt: TRDALGO_001)"
+
+  echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
+  echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+  echo -n " [-a <admin_port (7224)>"]
+  echo
+  exit 1
+}
+
+args=${*}
+
+ConfigServer=cloud23.cvtt.vpn:6789
+ActiveExchanges=ALPACA_SNDBX-MDPORTAL
+AdminPort=7224
+Strategy=DAILY_STOCK_001
+Book=""
+
+
+while getopts ":b:c:e:a:S:" opt; do
+  case ${opt} in
+    c )
+      ConfigServer=$OPTARG
+      ;;
+    e )
+      ActiveExchanges=$OPTARG
+      ;;
+    a )
+      AdminPort=$OPTARG
+      ;;
+    b )
+      Book=$OPTARG
+      ;;
+    S )
+      Strategy=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+echo "ConfigServer=${ConfigServer}"
+echo "ActiveExchanges=${ActiveExchanges}"
+echo "Book=${Book}"
+echo "Strategy=${Strategy}"
+echo "AdminPort=${AdminPort}"
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+
+ImageName=quant
+
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/${ImageName}
+
+Cmd="docker run"
+Cmd="${Cmd} --detach"
+Cmd="${Cmd} --restart=unless-stopped"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ImageName}.${Book}"
+Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${args}"
+
+echo $Cmd
+eval $Cmd
@@ -0,0 +1,74 @@
+#!/bin/bash
+
+usage() {
+  echo -n "Usage: $0"
+  echo -n " -b <book>"
+  echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
+  echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+  echo -n " [-a <admin_port (7223)>"]
+  echo
+  exit 1
+}
+
+
+args=${*}
+
+ConfigServer=cloud23.cvtt.vpn:6789
+ActiveExchanges=ALPACA_SNDBX-MDPORTAL
+AdminPort=7223
+Book=""
+
+
+while getopts ":b:c:e:a:" opt; do
+  case ${opt} in
+    c )
+      ConfigServer=$OPTARG
+      ;;
+    e )
+      ActiveExchanges=$OPTARG
+      ;;
+    a )
+      AdminPort=$OPTARG
+      ;;
+    b )
+      Book=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+
+echo "ConfigServer=${ConfigServer}"
+echo "ActiveExchanges=${ActiveExchanges}"
+echo "Book=${Book}"
+echo "AdminPort=${AdminPort}"
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+
+ImageName=risk_mgr
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/${ImageName}
+
+Cmd="docker run"
+Cmd="${Cmd} --detach"
+Cmd="${Cmd} --restart=unless-stopped"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ImageName}.${Book}"
+Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${args}"
+
+echo $Cmd
+eval $Cmd
@@ -0,0 +1,87 @@
+#!/bin/bash
+
+usage() {
+  echo -n "Usage: $0"
+  echo -n " -b <book>"
+
+  echo -n " -A <algo> (dflt: TRDALGO_001)"
+
+  echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
+  echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+  echo -n " [-a <admin_port (7223)>"]
+  echo
+  exit 1
+}
+
+
+args=${*}
+
+# ConfigServer=cloud16.cvtt.vpn
+ConfigServer=cloud23.cvtt.vpn:6789
+ActiveExchanges=ALPACA_SNDBX-MDPORTAL
+AdminPort=7226
+Algo=TRDALGO_001
+Book=""
+
+
+while getopts ":b:c:e:a:" opt; do
+  case ${opt} in
+    c )
+      ConfigServer=$OPTARG
+      ;;
+    e )
+      ActiveExchanges=$OPTARG
+      ;;
+    a )
+      AdminPort=$OPTARG
+      ;;
+    b )
+      Book=$OPTARG
+      ;;
+    A )
+      Algo=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+echo "ConfigServer=${ConfigServer}"
+echo "ActiveExchanges=${ActiveExchanges}"
+echo "Book=${Book}"
+echo "Algo=${Algo}"
+echo "AdminPort=${AdminPort}"
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+
+ImageName=trader
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/${ImageName}
+
+Cmd="docker run"
+Cmd="${Cmd} --detach"
+Cmd="${Cmd} --restart=unless-stopped"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ImageName}.${Book}"
+Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${args}"
+
+echo $Cmd
+eval $Cmd
@@ -0,0 +1,81 @@
+#!/bin/bash
+
+usage() {
+  echo -n "Usage: $0"
+  echo -n " -b <book>"
+  echo -n " [-C <config (dflt: apps/cvtt_eqt_alpaca)>]"
+  echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
+  echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+  echo -n " [-a <admin_port (7223)>"]
+  echo
+  exit 1
+}
+
+
+args=${*}
+
+# ConfigServer=cloud16.cvtt.vpn
+ConfigServer=cloud23.cvtt.vpn:6789
+ActiveExchanges=ALPACA_SNDBX-MDPORTAL
+AdminPort=7225
+Book=""
+Config=apps/cvtt_eqt_alpaca
+
+while getopts ":C:b:c:e:a:" opt; do
+  case ${opt} in
+    c )
+      ConfigServer=$OPTARG
+      ;;
+    e )
+      ActiveExchanges=$OPTARG
+      ;;
+    a )
+      AdminPort=$OPTARG
+      ;;
+    b )
+      Book=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+echo "Config=${Config}"
+echo "ConfigServer=${ConfigServer}"
+echo "ActiveExchanges=${ActiveExchanges}"
+echo "Book=${Book}"
+echo "AdminPort=${AdminPort}"
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+
+ImageName=trading_recorder
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/${ImageName}
+
+Cmd="docker run"
+Cmd="${Cmd} --detach"
+Cmd="${Cmd} --restart=unless-stopped"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ImageName}.${Book}"
+Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${args}"
+
+echo $Cmd
+eval $Cmd
@@ -1 +1 @@
-1.5.1A,md checklists
+1.6.2F2,load eqt to research - fix
@@ -9,6 +9,7 @@ usage() {
  echo
  exit 1
 }
+
 if [[ "$(uname)" == "Darwin" ]]; then
    # macOS
    date='gdate'
@@ -22,6 +23,8 @@ host=homestore
 stocks=COIN,GBTC,SQ

 TargetDir="/opt/jupyter_gpu/data/eqty_md"
+# TargetDir="/tmp"  # -------= D E B U G
+
 mkdir -p ${TargetDir}

 Table=md_1min_bars
@@ -88,8 +91,6 @@ TargetFile=$(${date} -d ${md_date} "+%Y%m%d.eqty.mktdata.ohlcv.db")

 TargetFilePath="${TargetDir}/${TargetFile}"
 mv ${TargetFilePath} "${TargetFilePath}.saved.$(${date} '+%Y%m%d_%H%M%S')"
-touch ${TargetFilePath}
-

 # ---- temp dir
 echo ${SourceFile}
@@ -136,11 +137,19 @@ function download_file {
    Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/"
    echo ${Cmd}
    eval ${Cmd}
+
+    if [[ ! -f ${tmp_dir}/${SourceFileZip} ]] ; then
+      echo "File ${SourceUser}@${SourceHost}:${SourceFilePath} is missing. Skipped."
+      return
+    fi
+
    ls -lh ${tmp_dir}
    Cmd="gunzip ${tmp_dir}/${SourceFileZip}"
    echo ${Cmd} && eval ${Cmd}

    DbFile=${tmp_dir}/${SourceFile}
+    touch ${TargetFilePath}
+

 # --- T E M P - for files older than 20240813
    sqlite3 ${DbFile} <<EOF
@@ -149,7 +158,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_idx on md_1min_bars(tstamp, excha
 EOF
 # --- T E M P

-    Cmd="sqlite3 ${DbFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}"
+    Cmd="sqlite3 ${DbFile} \".dump\" | sqlite3 ${TargetFilePath}"
    echo ${Cmd}
    eval ${Cmd}

@@ -167,8 +176,10 @@ for stock in "${Stocks[@]}"; do
    download_file ${SourceFilePath}
 done

-chmod 600 ${TargetFilePath}
+if [ -f ${TargetFilePath} ]; then
+    chmod 600
    ls -lh ${TargetFilePath}
+fi

 # mkdir -p /tmp/aaa
 # cp -r ${tmp_dir}/* /tmp/aaa/
Author	SHA1	Message	Date
oleg	b61c3a684a	load eqt to research - fix	2024-08-28 10:49:07 -04:00
oleg	78382df12d	load eqt to research - fix	2024-08-26 16:59:01 -04:00
oleg	bbfb7112bb	trading recorder -> docker	2024-08-24 19:19:18 -04:00
oleg	23603ad076	md_portal uses different config	2024-08-24 18:23:27 -04:00
oleg	81fe6372e9	added quant docker start script	2024-08-22 18:36:29 -04:00
oleg	48f77e5280	added quant docker start script	2024-08-22 18:31:14 -04:00
oleg	2c88bc613c	added trader docker start script	2024-08-22 15:59:02 -04:00
oleg	085f1af3da	added trader docker start script	2024-08-22 15:52:05 -04:00
oleg	fa93450b3d	fix executor start	2024-08-20 21:55:13 -04:00
oleg	4188dd7c7c	fix executor start	2024-08-20 21:50:54 -04:00
oleg	929953c1c4	fix executor start	2024-08-20 21:48:45 -04:00
oleg	e562716b73	renamed	2024-08-20 21:45:52 -04:00
oleg	7c806107ca	executor start added	2024-08-20 21:26:00 -04:00
oleg	85c4042d00	md_portal docker start	2024-08-19 13:00:44 -04:00
oleg	dcca83a3ae	trading: risk manager docker initial	2024-08-18 21:25:14 -04:00
oleg	e192766c9a	load EQT md to GPUshnik fix	2024-08-18 17:31:33 -04:00