rearrange + md monitor

2024-07-29 20:39:23 -04:00
parent da15fba6a4
commit 720f588ed2
19 changed files with 157 additions and 31 deletions
@@ -0,0 +1,7 @@
+{
+    "__dummy__": null
+    , "ALPACA_SANDBOX": {
+        "api_key": "PKLZSLFZMFMN1R28K9HK"
+        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
+    }
+}
@@ -0,0 +1,43 @@
+FROM python:3.10-slim
+
+ARG FROM_DIR=docker_dev/market_data/alpaca_md_day
+
+# Update the package list and install required packages
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    apt-utils \
+    software-properties-common \
+    curl \
+    rsync \
+    openssh-client \
+    parallel \
+    jq
+
+RUN pip install --upgrade pip
+
+
+COPY ${FROM_DIR}/requirements.txt /
+RUN pip install -r /requirements.txt
+
+COPY cvttpy /cvttpy
+COPY ${FROM_DIR}/.creds  /.creds
+COPY ${FROM_DIR}/.creds  /root/.creds
+COPY ${FROM_DIR}/alpaca_md_day.sh /alpaca_md_day.sh
+
+COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
+COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
+COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
+
+RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
+
+
+# Shared Volumes
+RUN mkdir -p /app/data
+RUN mkdir /logs
+RUN chmod +x /alpaca_md_day.sh
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENV PYTHONPATH=/
+ENTRYPOINT [ "/alpaca_md_day.sh" ]
+
+# CMD [ "echo", "alpaca_md_day"]
@@ -0,0 +1,116 @@
+# Build/Dev Host
+## Build
+```bash
+Version=$(cat /home/oleg/develop/cvtt2/cvttpy/release_version.txt)
+
+cd /home/oleg/develop/cvtt2
+docker build  -t crypto_md_day -t crypto_md_day:${Version} -f docker_dev/crypto_md_day/Dockerfile .
+
+## Deploy
+
+/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day
+
+```
+
+## On target Machine make sure user can use docker
+```bash
+sudo usermod -a -G docker cvtt
+```
+**Re-login as cvtt after adding cvtt to the group**
+
+### Clean
+```bash
+docker rm -f crypto_md_day
+docker images --filter=reference=crypto_md_day --format="{{.ID}}" | uniq | xargs docker rmi -f
+```
+### Test run locally
+```
+docker load -i /home/oleg/docker_images/crypto_md_day.img.tar
+mkdir -p /tmp/data
+mkdir -p /tmp/logs
+docker run --network="host" --name=crypto_md_day -v /tmp/data:/app/data -v /tmp/logs:/logs crypto_md_day
+```
+
+## Restarting
+```bash
+docker restart crypto_md_day
+```
+
+# Useful Commands
+```bash
+docker images
+docker ps -a
+docker image rm <image>
+docker rm <container>
+docker exec -it <container> /bin/bash
+```
+
+---
+```bash
+docker rm -f crypto_md_day
+docker images --filter=reference=crypto_md_day --format="{{.ID}}" | uniq | xargs docker rmi -f
+
+/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day cryptoval4.cvtt.vpn
+docker tag crypto_md_day:latest cloud21.cvtt.vpn:5500/crypto_md_day:latest
+docker push cloud21.cvtt.vpn:5500/crypto_md_day:latest
+
+# Local test
+docker load -i /home/oleg/docker_images/crypto_md_day.img.tar  && docker run --network="host" --name=crypto_md_day -v /tmp/data:/app/data crypto_md_day -h cvttdata -d 20240717 -s coinbase
+```
+
+## Local (cloud21) docker registry
+### Run registry container (cloud21)
+
+
+```bash
+mkdir /opt/store/cvtt/docker_registry/data
+vi /opt/store/cvtt/docker_registry/config.yml
+```
+
+```yaml
+version: 0.1
+log:
+  fields:
+    service: registry
+storage:
+  filesystem:
+    rootdirectory: /var/lib/registry
+http:
+  addr: :5500
+
+```
+```bash
+docker run -d -p 5500:5500 --name registry -v /opt/store/cvtt/docker_registry/config.yml:/etc/docker/registry/config.yml -v /opt/store/cvtt/docker_registry/data:/var/lib/registry registry:2
+```
+
+### Building
+#### /etc/docker/daemon.json
+```json
+{
+  "insecure-registries": ["cloud21.cvtt.vpn:5500"]
+}
+```
+```bash
+sudo systemctl restart docker
+```
+```bash
+/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day cryptoval4.cvtt.vpn
+docker tag crypto_md_day:latest cloud21.cvtt.vpn:5500/crypto_md_day:latest
+docker push cloud21.cvtt.vpn:5500/crypto_md_day:latest
+```
+
+### Loading and Running (cryptoval4)
+#### /etc/docker/daemon.json
+```json
+{
+  "insecure-registries": ["cloud21.cvtt.vpn:5500"]
+}
+```
+```bash
+sudo systemctl restart docker
+```
+```
+docker pull cloud21.cvtt.vpn:5500/crypto_md_day:latest
+docker run --network="host" --name=cloud21.cvtt.vpn:5500/crypto_md_day crypto_md_day -h cvttdata -d 20240717 -s coinbase
+```
+
@@ -0,0 +1,138 @@
+#!/bin/bash
+
+# --- Settings
+export PYTHONPATH=/
+export Python=python3.10
+export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
+export PyScript=/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
+
+export OutputDir=/app/data/alpaca_md # Local
+export LogDir=/logs
+
+DEFAULT_NUM_JOBS=10
+SLEEP_SEC_AFTER_PROC=2
+DEFAULT_SLICE_SIZE=500
+# --- Settings
+
+mkdir -p ${OutputDir}
+mkdir -p ${LogDir}
+
+
+usage() {
+  echo "Usage: $0 <date (YYYY-MM-DD)> [<num-jobs> (30) ] [<instrument_list_file>] "
+  exit 1
+}
+
+echo "CommandLine: ${*}"
+
+Start=${1}
+NumJobs=${2}
+
+export CalendarURL=http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+
+is_business_day() {
+    dt=${1}
+
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+
+if [ -z ${Start}  ] ; then
+    echo "start is not specified, getting yesterday..."
+    Start=$(date -d "yesterday" "+%Y-%m-%d")
+    echo "Start=${Start}"
+fi
+
+while true; do
+    if is_business_day ${Start}; then
+        break
+    fi
+    echo "${Start} is not business day in US"
+    Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
+done
+
+End=${Start}   # only 1 day is practical
+
+if [ "${NumJobs}" == "" ] ; then
+    NumJobs=${DEFAULT_NUM_JOBS}
+fi
+
+echo "Start=${Start} End=${End} NumJobs=${NumJobs}"
+
+
+run_proc() {
+    Inst=${1}
+    Start=${2}
+    End=${3}
+    echo "Running for $Inst"
+
+    Cmd="${Python} ${PyScript}"
+    Cmd="${Cmd} --config=${Config}"
+    Cmd="${Cmd} --output_dir=${OutputDir}"
+    Cmd="${Cmd} --instruments=ALPACA:${Inst}"
+    Cmd="${Cmd} --start=${Start}"
+    if [ "${End}" != "" ]; then
+        Cmd="${Cmd} --end=${End}"
+    fi
+    Cmd="${Cmd} --log_file=${LogDir}/${Inst}.log"
+    Cmd="${Cmd} --log_level=WARNING"
+    echo ${Cmd}
+    eval ${Cmd}
+
+    sleep 2 #${SLEEP_SEC_AFTER_PROC}
+}
+
+export -f run_proc
+
+key=$(jq -r '.["ALPACA_SANDBOX"] | .api_key' ~/.creds)
+secret=$(jq -r '.["ALPACA_SANDBOX"] | .secret_key' ~/.creds)
+
+Cmd="curl -s --request GET --url 'https://paper-api.alpaca.markets/v2/assets?status=active'"
+Cmd="${Cmd} --header 'APCA-API-KEY-ID: ${key}'"
+Cmd="${Cmd} --header 'APCA-API-SECRET-KEY: ${secret}'"
+Cmd="${Cmd} --header 'accept: application/json'"
+Cmd="${Cmd} | jq '.[] | select(.class == \"us_equity\" and .exchange != \"OTC\") | .symbol'"
+Cmd="${Cmd}  | sed 's/\"//g'"
+Cmd="${Cmd} | sed 's/^/STOCK-/'"
+# split string into array
+Instruments=()
+for Inst in $(eval ${Cmd})
+do
+    Instruments+=("$Inst")
+done
+
+slice_size=${DEFAULT_SLICE_SIZE} # 10K symbols parallel cannot handle
+for ((ii=0; ii <${#Instruments[@]}; ii+=slice_size)); do
+    InstSlice=("${Instruments[@]:ii:slice_size}")
+
+    parallel -j ${NumJobs} run_proc {} ${Start} ${End} ::: "${InstSlice[@]}"
+done
+
+echo "Compressing"
+for file in $(find ${OutputDir} -type f -name '*db' -print )
+do
+    echo "Compressing ${file}"
+    gzip ${file}
+done
+
+
+Source=/app/data/
+Targets=
+Targets="${Targets} oleg@homestore.cvtt.vpn:/works/cvtt/md_archive/equity"
+Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
+
+
+for tgt in ${Targets}
+do
+    Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
+    echo $Cmd
+    eval $Cmd
+done
+
@@ -0,0 +1,7 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5
+asyncpg>=0.27.0
+hjson>=3.1.0
+pandas>=1.5.3
+sortedcontainers>=2.4.0
+aioredis>=2.0.1