progress

2025-05-17 19:28:56 -04:00
parent 4e670be142
commit e083eb27d9
5 changed files with 0 additions and 0 deletions
@@ -0,0 +1,3 @@
+.git
+**/__pycache__
+.pipenv
@@ -0,0 +1,34 @@
+FROM python:3.12-slim
+
+ARG FROM_DIR=docker_dev/market_data/crypto_md_day
+
+COPY ${FROM_DIR}/requirements.txt /
+
+
+COPY cvttpy /cvttpy
+COPY ${FROM_DIR}/crypto_md_day.sh /crypto_md_day.sh
+
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+RUN apt-get update && apt-get install -y \
+    libpq-dev \
+    build-essential
+RUN apt-get update && apt-get install -y rsync openssh-client
+
+COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
+COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
+
+RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub
+
+
+# Shared Volumes
+RUN mkdir -p /app/data
+RUN mkdir /logs
+RUN chmod +x /crypto_md_day.sh
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENV PYTHONPATH=/
+ENTRYPOINT [ "/crypto_md_day.sh" ]
+
@@ -0,0 +1,116 @@
+# Build/Dev Host
+## Build
+```bash
+Version=$(cat /home/oleg/develop/cvtt2/cvttpy/VERSION)
+
+cd /home/oleg/develop/cvtt2
+docker build  -t crypto_md_day -t crypto_md_day:${Version} -f docker_dev/crypto_md_day/Dockerfile .
+
+## Deploy
+
+/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day
+
+```
+
+## On target Machine make sure user can use docker
+```bash
+sudo usermod -a -G docker cvtt
+```
+**Re-login as cvtt after adding cvtt to the group**
+
+### Clean
+```bash
+docker rm -f crypto_md_day
+docker images --filter=reference=crypto_md_day --format="{{.ID}}" | uniq | xargs docker rmi -f
+```
+### Test run locally
+```
+docker load -i /home/oleg/docker_images/crypto_md_day.img.tar
+mkdir -p /tmp/data
+mkdir -p /tmp/logs
+docker run --network="host" --name=crypto_md_day -v /tmp/data:/app/data -v /tmp/logs:/logs crypto_md_day
+```
+
+## Restarting
+```bash
+docker restart crypto_md_day
+```
+
+# Useful Commands
+```bash
+docker images
+docker ps -a
+docker image rm <image>
+docker rm <container>
+docker exec -it <container> /bin/bash
+```
+
+---
+```bash
+docker rm -f crypto_md_day
+docker images --filter=reference=crypto_md_day --format="{{.ID}}" | uniq | xargs docker rmi -f
+
+/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day cryptoval4.cvtt.vpn
+docker tag crypto_md_day:latest cloud21.cvtt.vpn:5500/crypto_md_day:latest
+docker push cloud21.cvtt.vpn:5500/crypto_md_day:latest
+
+# Local test
+docker load -i /home/oleg/docker_images/crypto_md_day.img.tar  && docker run --network="host" --name=crypto_md_day -v /tmp/data:/app/data crypto_md_day -h cvttdata -d 20240717 -s coinbase
+```
+
+## Local (cloud21) docker registry
+### Run registry container (cloud21)
+
+
+```bash
+mkdir /opt/store/cvtt/docker_registry/data
+vi /opt/store/cvtt/docker_registry/config.yml
+```
+
+```yaml
+version: 0.1
+log:
+  fields:
+    service: registry
+storage:
+  filesystem:
+    rootdirectory: /var/lib/registry
+http:
+  addr: :5500
+
+```
+```bash
+docker run -d -p 5500:5500 --name registry -v /opt/store/cvtt/docker_registry/config.yml:/etc/docker/registry/config.yml -v /opt/store/cvtt/docker_registry/data:/var/lib/registry registry:2
+```
+
+### Building
+#### /etc/docker/daemon.json
+```json
+{
+  "insecure-registries": ["cloud21.cvtt.vpn:5500"]
+}
+```
+```bash
+sudo systemctl restart docker
+```
+```bash
+/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day cryptoval4.cvtt.vpn
+docker tag crypto_md_day:latest cloud21.cvtt.vpn:5500/crypto_md_day:latest
+docker push cloud21.cvtt.vpn:5500/crypto_md_day:latest
+```
+
+### Loading and Running (cryptoval4)
+#### /etc/docker/daemon.json
+```json
+{
+  "insecure-registries": ["cloud21.cvtt.vpn:5500"]
+}
+```
+```bash
+sudo systemctl restart docker
+```
+```
+docker pull cloud21.cvtt.vpn:5500/crypto_md_day:latest
+docker run --network="host" --name=cloud21.cvtt.vpn:5500/crypto_md_day crypto_md_day -h cvttdata -d 20240717 -s coinbase
+```
+
@@ -0,0 +1,83 @@
+#!/bin/bash
+
+
+usage() {
+  echo "Usage: $0 -h <db host> -d <YYYYMMDD Date> -s <schemas|coinbase,bnbspot,bnbfut>"
+  exit 1
+}
+
+DbHost=""
+Date=""
+# Schemas=coinbase,bnbspot,bnbfut
+Schemas=coinbase,bnbspot
+
+while getopts ":h:d:s:" opt; do
+  case ${opt} in
+    h )
+      DbHost=$OPTARG
+      ;;
+    d )
+      Date=$OPTARG
+      ;;
+    s )
+      Schemas=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+if [ -z "${DbHost}" ] || [ -z "$Date" ]; then
+  echo "DbHost=${DbHost} Date=${Date}. Cmdline: ${*}"
+  usage
+fi
+
+CredKey=""
+if [ "${DbHost}" == "cloud28" ] ; then
+    CredKey=TSDB_MD_CLOUD28_RO
+elif [ "${DbHost}" == "cloud21" ] ; then
+    CredKey=TSDB_MD_CLD21_RO
+else
+    echo "host ${DbHost} is not valid source"
+    usage
+fi
+
+export PYTHONPATH=/
+
+Cmd="python3.12"
+Cmd+=" cvttpy/research/utils/archive_ts_md.py"
+Cmd+=" --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder"
+Cmd+=" --db_credentials_key=${CredKey}"
+Cmd+=" --credentials_file=/.creds"
+Cmd+=" --date=${Date}"
+Cmd+=" --schemas=${Schemas}"
+Cmd+=" --root_dir=/app/data/${DbHost}"
+Cmd+=" --format=SQLite"
+Cmd+=" --compress"
+# Cmd+=" --log_file=/logs/%T.crypto_md_day.log"
+
+echo ${Cmd}
+eval ${Cmd}
+
+echo "Data Sync"
+
+Source=/app/data/
+Targets=
+Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto"
+Targets="${Targets} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto"
+
+for tgt in ${Targets}
+do
+    Cmd="/usr/bin/rsync -ahv"
+    Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
+    Cmd+=" ${Source} ${tgt}"
+    echo $Cmd
+    eval $Cmd
+done
+
@@ -0,0 +1,6 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5
+psycopg>=3.2.1
+hjson>=3.1.0
+pandas>=1.5.3
+sortedcontainers>=2.4.0