VERSION

@@ -1 +0,0 @@
-0.2.7,new md_recorder build

__TBD__/buildx.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+echo "Script $0 does not work yet. Aborted" && exit
+
+function usage {
+    echo "Usage: ${0} <app_path (e.g. market_data/md_recorder_monitor)> <version>"
+    exit
+}
+
+AppPath=${1}
+CvttpyVersion=${2}
+
+if [ "${AppPath}" == "" ]
+then
+    usage
+fi
+
+# --- Settings
+DockerDir=$(realpath $(dirname ${0})/${AppPath})
+RootDir=$(realpath $(dirname ${0})/..)
+RegistryService=cloud21.cvtt.vpn:5500
+
+AppName=$(basename ${AppPath})
+
+ImageName=${AppName}
+ImageDir=${HOME}/docker_images
+mkdir -p ${ImageDir}
+
+Version=$(cat ${RootDir}/cvttpy/release_version.txt | awk -F, '{print $1}')
+
+cd ${RootDir}
+
+# Create a new builder instance
+Cmd="docker buildx create --name ${AppName}_builder"
+echo ${Cmd}
+eval ${Cmd} || exit
+
+# Switch to the newly created builder
+Cmd="docker buildx use ${AppName}_builder"
+echo ${Cmd}
+eval ${Cmd}
+
+# Ensure buildx is bootstrapped
+Cmd="docker buildx inspect ${AppName}_builder --bootstrap"
+echo ${Cmd}
+eval ${Cmd}
+
+# Build the image using buildx, tagging with both 'latest' and the version number, and pushing directly
+Cmd="docker buildx build --platform linux/amd64,linux/arm64 -t ${ImageName}:latest -t ${ImageName}:${Version} -f ${DockerDir}/Dockerfile . --push"
+echo ${Cmd}
+eval ${Cmd}
+
+# Tagging the images with the registry
+Cmd="docker tag ${ImageName}:latest ${RegistryService}/${ImageName}:latest"
+echo ${Cmd}
+eval ${Cmd}
+
+Cmd="docker tag ${ImageName}:${Version} ${RegistryService}/${ImageName}:${Version}"
+echo ${Cmd}
+eval ${Cmd}
+
+# Push the images to the registry
+Cmd="docker push ${RegistryService}/${ImageName}:latest"
+echo ${Cmd}
+eval ${Cmd}
+
+Cmd="docker push ${RegistryService}/${ImageName}:${Version}"
+echo ${Cmd}
+eval ${Cmd}
+
+# Optionally, remove the builder to clean up
+Cmd="docker buildx rm ${AppName}_builder"
+echo ${Cmd}
+eval ${Cmd}

build.sh

@@ -25,7 +25,7 @@ ImageDir=${HOME}/docker_images
 mkdir -p ${ImageDir}
 
 
-Version=$(cat ${RootDir}/${Project}/VERSION | awk -F, '{print $1}')
+Version=$(cat ${RootDir}/${Project}/release_version.txt | awk -F, '{print $1}')
 # Version=${ProjectVersion}
 
 cd ${RootDir}

config_service/Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.12-slim
+
+WORKDIR /
+COPY docker_dev/config_service/requirements.txt /
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+
+# Shared Volumes
+RUN mkdir -p /app/data
+RUN mkdir /logs
+
+ENV PYTHONPATH=/
+CMD [                                        \
+    "python3.12",                            \
+    "cvttpy/apps/utils/config_server.py",    \
+    "--port=6789",                           \
+    "--root=/app/data",                      \
+    "--log_file=/logs/%T.config_service.log" \
+]
+

config_service/HOWTO.md

@@ -0,0 +1,42 @@
+# Build/Dev Host
+## Build
+```bash
+TgtHostname=cloud16.cvtt.vpn
+Version=$(cat cvttpy/release_version.txt)
+
+cd /home/oleg/develop/cvtt2
+docker build  -t config_service -t config_service:${Version} -f cvttpy/utils/docker/config_service/Dockerfile .
+
+## Deploy
+
+docker save -o /tmp/cvtt_config_service.img.tar config_service
+scp /tmp/cvtt_config_service.img.tar cvtt@${TgtHostname}:/tmp/
+
+ssh cvtt@${TgtHostname}
+```
+
+## On target Machine make sure user can use docker
+```bash
+sudo usermod -a -G docker cvtt
+```
+**Re-login as cvtt after adding cvtt to the group**
+
+```bash
+docker rm -f cvtt_config_service
+docker load -i /tmp/cvtt_config_service.img.tar
+docker run -d --name=cvtt_config_service -p 6789:6789 -v /home/cvtt/prod/config_service/data:/app/data -v /home/cvtt/prod/logs:/logs config_service
+```
+
+## Restarting
+```bash
+docker restart cvtt_config_service
+```
+
+# Useful Commands
+```bash
+docker images
+docker ps -a
+docker image rm <image>
+docker rm <container>
+docker exec -it <container> /bin/bash
+```

config_service/requirements.txt

@@ -0,0 +1,2 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5

dckr_bld_functions.sh

@@ -1,76 +0,0 @@
-#!/bin/bash
-
-get_project_version() {
-    Project=${1}
-    Version=${2}
-
-    host="cloud21.cvtt.vpn"
-    port="22"
-    user="cvttdist"
-    rel_dir="/home/cvttdist/software/cvtt2"
-
-    if [ "${Version}" == "latest" ]; then
-        echo "Checking for latest version of ${Project} on ${user}@${host}:${rel_dir}"
-        Version=$(ssh -q -p ${port} ${user}@${host} "ls -tr ${rel_dir}/${Project} | tail -1" )
-        echo "Latest version is ${Version}"
-    fi
-    echo "Checking ${user}@${host} for ${rel_dir}/${Project}/${Version} ..."
-    if ssh -q -p ${port} ${user}@${host} "test -d ${rel_dir}/${Project}/${Version}"
-    then
-        echo "Getting Version \"${Version}\"..."
-        rsync_cmd="rsync -ahv -e \"ssh -p ${port}\""
-        rsync_cmd="${rsync_cmd} ${user}@${host}:${rel_dir}/${Project}/${Version}/"
-        rsync_cmd="${rsync_cmd} ./"
-        echo ${rsync_cmd}
-        eval ${rsync_cmd}
-        status=$?
-        if [ ${status} -eq 0 ]
-        then
-            echo "Loading successful..."
-            return 0
-        else
-            return 1
-        fi
-    else
-        echo "Not Found ${rel_dir}/${Project}/${Version} on ${user}@${host}"
-        return 1
-    fi
-}
-
-build_docker_image() {
-    ImageName=${1}
-    RegistryService=${2}
-    Version=${3}
-    Project=${4}
-    ProjectVersion=${5}
-
-    Cmd="docker build"
-    if [ "${ProjectVersion}" != "" ]; then
-        Cmd+=" --label ${Project}=\"${ProjectVersion}\""
-    fi
-    Cmd+=" -t ${ImageName}"
-    Cmd+=" -t ${ImageName}:latest"
-    Cmd+=" -t ${ImageName}:${Version}"
-    Cmd+=" -f Dockerfile"
-    Cmd+=" ."
-    echo ${Cmd}
-    eval ${Cmd} || exit
-
-    Cmd="docker tag ${ImageName}:latest ${RegistryService}/${ImageName}:latest"
-    echo ${Cmd}
-    eval ${Cmd} || exit
-
-    Cmd="docker tag ${ImageName}:${Version} ${RegistryService}/${ImageName}:${Version}"
-    echo ${Cmd}
-    eval ${Cmd} || exit
-
-    Cmd="docker push ${RegistryService}/${ImageName}:latest"
-    echo ${Cmd}
-    eval ${Cmd} || exit
-
-    Cmd="docker push ${RegistryService}/${ImageName}:${Version}"
-    echo ${Cmd}
-    eval ${Cmd} || exit
-}
-
-

market_data/_OLD_/alpaca_md_day/.creds

@@ -0,0 +1,7 @@
+{
+    "__dummy__": null
+    , "ALPACA_SANDBOX": {
+        "api_key": "PKLZSLFZMFMN1R28K9HK"
+        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
+    }
+}

market_data/_OLD_/alpaca_md_day/Dockerfile

@@ -0,0 +1,46 @@
+FROM python:3.12-slim
+
+ARG FROM_DIR=docker_dev/market_data/alpaca_md_day
+
+# Update the package list and install required packages
+RUN apt-get update && apt-get install -y \
+    apt-utils \
+    libpq-dev \
+    build-essential
+RUN apt-get update && apt-get install -y rsync openssh-client
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    software-properties-common \
+    curl \
+    parallel \
+    jq
+
+RUN pip install --upgrade pip
+
+
+COPY ${FROM_DIR}/requirements.txt /
+RUN pip install -r /requirements.txt
+
+COPY cvttpy /cvttpy
+COPY ${FROM_DIR}/.creds  /.creds
+COPY ${FROM_DIR}/.creds  /root/.creds
+COPY ${FROM_DIR}/alpaca_md_day.sh /alpaca_md_day.sh
+
+COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
+COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
+COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
+
+RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
+
+
+# Shared Volumes
+RUN mkdir -p /app/data
+RUN mkdir /logs
+RUN chmod +x /alpaca_md_day.sh
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENV PYTHONPATH=/
+ENTRYPOINT [ "/alpaca_md_day.sh" ]
+
+# CMD [ "echo", "alpaca_md_day"]

market_data/_OLD_/alpaca_md_day/HOWTO.md

@@ -1,7 +1,7 @@
 # Build/Dev Host
 ## Build
 ```bash
-Version=$(cat /home/oleg/develop/cvtt2/cvttpy/VERSION)
+Version=$(cat /home/oleg/develop/cvtt2/cvttpy/release_version.txt)
 
 cd /home/oleg/develop/cvtt2
 docker build  -t crypto_md_day -t crypto_md_day:${Version} -f docker_dev/crypto_md_day/Dockerfile .

market_data/_OLD_/alpaca_md_day/alpaca_md_day.sh

@@ -0,0 +1,138 @@
+#!/bin/bash
+
+# --- Settings
+export PYTHONPATH=/
+export Python=python3.12
+export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
+export PyScript=/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
+
+export OutputDir=/app/data/alpaca_md # Local
+export LogDir=/logs
+
+DEFAULT_NUM_JOBS=10
+SLEEP_SEC_AFTER_PROC=2
+DEFAULT_SLICE_SIZE=500
+# --- Settings
+
+mkdir -p ${OutputDir}
+mkdir -p ${LogDir}
+
+
+usage() {
+  echo "Usage: $0 <date (YYYY-MM-DD)> [<num-jobs> (30) ] [<instrument_list_file>] "
+  exit 1
+}
+
+echo "CommandLine: ${*}"
+
+Start=${1}
+NumJobs=${2}
+
+export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+
+is_business_day() {
+    dt=${1}
+
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+
+if [ -z ${Start}  ] ; then
+    echo "start is not specified, getting yesterday..."
+    Start=$(date -d "yesterday" "+%Y-%m-%d")
+    echo "Start=${Start}"
+fi
+
+while true; do
+    if is_business_day ${Start}; then
+        break
+    fi
+    echo "${Start} is not business day in US"
+    Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
+done
+
+End=${Start}   # only 1 day is practical
+
+if [ "${NumJobs}" == "" ] ; then
+    NumJobs=${DEFAULT_NUM_JOBS}
+fi
+
+echo "Start=${Start} End=${End} NumJobs=${NumJobs}"
+
+
+run_proc() {
+    Inst=${1}
+    Start=${2}
+    End=${3}
+    echo "Running for $Inst"
+
+    Cmd="${Python} ${PyScript}"
+    Cmd="${Cmd} --config=${Config}"
+    Cmd="${Cmd} --output_dir=${OutputDir}"
+    Cmd="${Cmd} --instruments=ALPACA:${Inst}"
+    Cmd="${Cmd} --start=${Start}"
+    if [ "${End}" != "" ]; then
+        Cmd="${Cmd} --end=${End}"
+    fi
+    Cmd="${Cmd} --log_file=${LogDir}/${Inst}.log"
+    Cmd="${Cmd} --log_level=WARNING"
+    echo ${Cmd}
+    eval ${Cmd}
+
+    sleep 2 #${SLEEP_SEC_AFTER_PROC}
+}
+
+export -f run_proc
+
+key=$(jq -r '.["ALPACA_SANDBOX"] | .api_key' ~/.creds)
+secret=$(jq -r '.["ALPACA_SANDBOX"] | .secret_key' ~/.creds)
+
+Cmd="curl -s --request GET --url 'https://paper-api.alpaca.markets/v2/assets?status=active'"
+Cmd="${Cmd} --header 'APCA-API-KEY-ID: ${key}'"
+Cmd="${Cmd} --header 'APCA-API-SECRET-KEY: ${secret}'"
+Cmd="${Cmd} --header 'accept: application/json'"
+Cmd="${Cmd} | jq '.[] | select(.class == \"us_equity\" and .exchange != \"OTC\") | .symbol'"
+Cmd="${Cmd}  | sed 's/\"//g'"
+Cmd="${Cmd} | sed 's/^/STOCK-/'"
+# split string into array
+Instruments=()
+for Inst in $(eval ${Cmd})
+do
+    Instruments+=("$Inst")
+done
+
+slice_size=${DEFAULT_SLICE_SIZE} # 10K symbols parallel cannot handle
+for ((ii=0; ii <${#Instruments[@]}; ii+=slice_size)); do
+    InstSlice=("${Instruments[@]:ii:slice_size}")
+
+    parallel -j ${NumJobs} run_proc {} ${Start} ${End} ::: "${InstSlice[@]}"
+done
+
+echo "Compressing"
+for file in $(find ${OutputDir} -type f -name '*db' -print )
+do
+    echo "Compressing ${file}"
+    gzip ${file}
+done
+
+
+Source=/app/data/
+Targets=
+Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity"
+Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
+
+
+for tgt in ${Targets}
+do
+    Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
+    echo $Cmd
+    eval $Cmd
+done
+

market_data/_OLD_/alpaca_md_day/requirements.txt

@@ -1,6 +1,6 @@
 aiohttp>=3.7.4.post0
 nest-asyncio>=1.5.5
-numpy>=1.24.1
+psycopg>=3.2.1
 hjson>=3.1.0
 pandas>=1.5.3
 sortedcontainers>=2.4.0

market_data/alpaca_hbar/.creds

@@ -0,0 +1,7 @@
+{
+    "__dummy__": null
+    , "ALPACA_SANDBOX": {
+        "api_key": "PKLZSLFZMFMN1R28K9HK"
+        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
+    }
+}

market_data/alpaca_hbar/Dockerfile

@@ -15,13 +15,14 @@ RUN apt-get update && apt-get install -y \
 # must be from disted version
 COPY cvtt-rust/bin/alpaca_md_hbar_loader /
 
-# COPY ${FROM_DIR}/.creds  /.creds
+COPY ${FROM_DIR}/.creds  /.creds
 COPY ${FROM_DIR}/alpaca_hbar.sh /alpaca_hbar.sh
 
 COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
 COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
+COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
 
-RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub
+RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
 
 
 # Shared Volumes

market_data/alpaca_hbar/alpaca_hbar.sh

@@ -43,6 +43,10 @@ while getopts ":d:N:" opt; do
   esac
 done
 
+# if [ -z ${Symbols}  ] ; then
+#     echo "symbols are not specified"
+#     usage
+# fi
 
 mkdir -p ${OutputDir}
 mkdir -p ${LogDir}
@@ -50,7 +54,7 @@ mkdir -p ${LogDir}
 
 echo "CommandLine: ${*}"
 
-export CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS
+export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
 
 
 if [ -z ${Start}  ] ; then
@@ -93,15 +97,13 @@ done
 
 Source=/data/
 Targets=
-Targets="${Targets} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity"
+Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity"
 Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
 
 
 for tgt in ${Targets}
 do
-    Cmd="/usr/bin/rsync -ahv"
-    Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
-    Cmd+=" ${Source} ${tgt}"
+    Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
     echo $Cmd
     eval $Cmd
 done

market_data/alpaca_qat/.creds

@@ -0,0 +1,7 @@
+{
+    "__dummy__": null
+    , "ALPACA_SANDBOX": {
+        "api_key": "PKLZSLFZMFMN1R28K9HK"
+        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
+    }
+}

market_data/alpaca_qat/Dockerfile

@@ -16,13 +16,14 @@ RUN apt-get update && apt-get install -y \
 # must be from disted version
 COPY cvtt-rust/bin/alpaca_md_qat_loader /
 
-# COPY ${FROM_DIR}/.creds  /.creds
+COPY ${FROM_DIR}/.creds  /.creds
 COPY ${FROM_DIR}/alpaca_qat.sh /alpaca_qat.sh
 
 COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
 COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
+COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
 
-RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub
+RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
 
 
 # Shared Volumes

market_data/alpaca_qat/alpaca_qat.sh

@@ -54,7 +54,7 @@ mkdir -p ${LogDir}
 
 echo "CommandLine: ${*}"
 
-export CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS
+export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
 
 
 if [ -z ${Start}  ] ; then
@@ -95,15 +95,13 @@ done
 
 Source=/data/
 Targets=
-Targets="${Targets} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity"
+Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity"
 Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
 
 
 for tgt in ${Targets}
 do
-    Cmd="/usr/bin/rsync -ahv"
-    Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
-    Cmd+=" ${Source} ${tgt}"
+    Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
     echo $Cmd
     eval $Cmd
 done

market_data/crypto_md_day/.creds

@@ -0,0 +1,17 @@
+{
+    "__dummy__": null
+     , "TSDB_MD_CLD21_RO": {
+         "host": "cloud21.cvtt.vpn"
+         , "port": 5432
+         , "user": "cvtt_ro"
+         , "database": "cvtt_md"
+         , "password": "3Gkotqa0ZSmZLXSH3RkTUm"
+     }
+    , "TSDB_MD_CVTTDATA_RO": {
+        "host": "cvttdata.cvtt.vpn"
+        , "port": 5432
+        , "user": "cvtt_ro"
+        , "database": "cvtt_md"
+        , "password": "3Gkotqa0ZSmZLXSH3RkTUm"
+    }
+}

market_data/crypto_md_day/Dockerfile

@@ -6,6 +6,7 @@ COPY ${FROM_DIR}/requirements.txt /
 
 
 COPY cvttpy /cvttpy
+COPY ${FROM_DIR}/.creds  /.creds
 COPY ${FROM_DIR}/crypto_md_day.sh /crypto_md_day.sh
 
 RUN pip install --upgrade pip --root-user-action=ignore
@@ -18,8 +19,9 @@ RUN apt-get update && apt-get install -y rsync openssh-client
 
 COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
 COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
+COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
 
-RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub
+RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
 
 
 # Shared Volumes

market_data/crypto_md_day/HOWTO.md

@@ -0,0 +1,116 @@
+# Build/Dev Host
+## Build
+```bash
+Version=$(cat /home/oleg/develop/cvtt2/cvttpy/release_version.txt)
+
+cd /home/oleg/develop/cvtt2
+docker build  -t crypto_md_day -t crypto_md_day:${Version} -f docker_dev/crypto_md_day/Dockerfile .
+
+## Deploy
+
+/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day
+
+```
+
+## On target Machine make sure user can use docker
+```bash
+sudo usermod -a -G docker cvtt
+```
+**Re-login as cvtt after adding cvtt to the group**
+
+### Clean
+```bash
+docker rm -f crypto_md_day
+docker images --filter=reference=crypto_md_day --format="{{.ID}}" | uniq | xargs docker rmi -f
+```
+### Test run locally
+```
+docker load -i /home/oleg/docker_images/crypto_md_day.img.tar
+mkdir -p /tmp/data
+mkdir -p /tmp/logs
+docker run --network="host" --name=crypto_md_day -v /tmp/data:/app/data -v /tmp/logs:/logs crypto_md_day
+```
+
+## Restarting
+```bash
+docker restart crypto_md_day
+```
+
+# Useful Commands
+```bash
+docker images
+docker ps -a
+docker image rm <image>
+docker rm <container>
+docker exec -it <container> /bin/bash
+```
+
+---
+```bash
+docker rm -f crypto_md_day
+docker images --filter=reference=crypto_md_day --format="{{.ID}}" | uniq | xargs docker rmi -f
+
+/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day cryptoval4.cvtt.vpn
+docker tag crypto_md_day:latest cloud21.cvtt.vpn:5500/crypto_md_day:latest
+docker push cloud21.cvtt.vpn:5500/crypto_md_day:latest
+
+# Local test
+docker load -i /home/oleg/docker_images/crypto_md_day.img.tar  && docker run --network="host" --name=crypto_md_day -v /tmp/data:/app/data crypto_md_day -h cvttdata -d 20240717 -s coinbase
+```
+
+## Local (cloud21) docker registry
+### Run registry container (cloud21)
+
+
+```bash
+mkdir /opt/store/cvtt/docker_registry/data
+vi /opt/store/cvtt/docker_registry/config.yml
+```
+
+```yaml
+version: 0.1
+log:
+  fields:
+    service: registry
+storage:
+  filesystem:
+    rootdirectory: /var/lib/registry
+http:
+  addr: :5500
+
+```
+```bash
+docker run -d -p 5500:5500 --name registry -v /opt/store/cvtt/docker_registry/config.yml:/etc/docker/registry/config.yml -v /opt/store/cvtt/docker_registry/data:/var/lib/registry registry:2
+```
+
+### Building
+#### /etc/docker/daemon.json
+```json
+{
+  "insecure-registries": ["cloud21.cvtt.vpn:5500"]
+}
+```
+```bash
+sudo systemctl restart docker
+```
+```bash
+/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day cryptoval4.cvtt.vpn
+docker tag crypto_md_day:latest cloud21.cvtt.vpn:5500/crypto_md_day:latest
+docker push cloud21.cvtt.vpn:5500/crypto_md_day:latest
+```
+
+### Loading and Running (cryptoval4)
+#### /etc/docker/daemon.json
+```json
+{
+  "insecure-registries": ["cloud21.cvtt.vpn:5500"]
+}
+```
+```bash
+sudo systemctl restart docker
+```
+```
+docker pull cloud21.cvtt.vpn:5500/crypto_md_day:latest
+docker run --network="host" --name=cloud21.cvtt.vpn:5500/crypto_md_day crypto_md_day -h cvttdata -d 20240717 -s coinbase
+```
+

market_data/crypto_md_day/crypto_md_day.sh

@@ -39,8 +39,8 @@ if [ -z "${DbHost}" ] || [ -z "$Date" ]; then
 fi
 
 CredKey=""
-if [ "${DbHost}" == "cloud28" ] ; then
-    CredKey=TSDB_MD_CLOUD28_RO
+if [ "${DbHost}" == "cvttdata" ] ; then
+    CredKey=TSDB_MD_CVTTDATA_RO
 elif [ "${DbHost}" == "cloud21" ] ; then
     CredKey=TSDB_MD_CLD21_RO
 else
@@ -51,16 +51,16 @@ fi
 export PYTHONPATH=/
 
 Cmd="python3.12"
-Cmd+=" cvttpy/research/utils/archive_ts_md.py"
-Cmd+=" --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder"
-Cmd+=" --db_credentials_key=${CredKey}"
-Cmd+=" --credentials_file=/.creds"
-Cmd+=" --date=${Date}"
-Cmd+=" --schemas=${Schemas}"
-Cmd+=" --root_dir=/app/data/${DbHost}"
-Cmd+=" --format=SQLite"
-Cmd+=" --compress"
-# Cmd+=" --log_file=/logs/%T.crypto_md_day.log"
+Cmd="${Cmd} cvttpy/research/utils/archive_ts_md.py"
+Cmd="${Cmd} --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder"
+Cmd="${Cmd} --db_credentials_key=${CredKey}"
+Cmd="${Cmd} --credentials_file=/.creds"
+Cmd="${Cmd} --date=${Date}"
+Cmd="${Cmd} --schemas=${Schemas}"
+Cmd="${Cmd} --root_dir=/app/data/${DbHost}"
+Cmd="${Cmd} --format=SQLite"
+Cmd="${Cmd} --compress"
+# Cmd="${Cmd} --log_file=/logs/%T.crypto_md_day.log"
 
 echo ${Cmd}
 eval ${Cmd}
@@ -69,14 +69,13 @@ echo "Data Sync"
 
 Source=/app/data/
 Targets=
+Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/crypto"
 Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto"
-Targets="${Targets} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto"
+
 
 for tgt in ${Targets}
 do
-    Cmd="/usr/bin/rsync -ahv"
-    Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
-    Cmd+=" ${Source} ${tgt}"
+    Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
     echo $Cmd
     eval $Cmd
 done

market_data/md_recorder/.creds

@@ -0,0 +1,17 @@
+{
+    "__dummy__": null
+    , "TSDB_MD_CLD21": {
+        "host": "cloud21.cvtt.vpn"
+        , "port": 5432
+        , "user": "cvtt"
+        , "database": "cvtt_md"
+        , "password": "ICdIh0JnMM7vM7Pf"
+    }
+    , "TSDB_MD_CVTTDATA": {
+        "host": "cvttdata.cvtt.vpn"
+        , "port": 5432
+        , "user": "cvtt"
+        , "database": "cvtt_md"
+        , "password": "ICdIh0JnMM7vM7Pf"
+    }
+}

market_data/md_recorder/.dockerignore

@@ -0,0 +1,3 @@
+.git
+**/__pycache__
+.pipenv

market_data/md_recorder/Dockerfile

@@ -0,0 +1,28 @@
+FROM python:3.12-slim
+
+ARG FROM_DIR=docker_dev/market_data/md_recorder
+
+RUN apt-get update && apt-get install -y \
+    apt-utils \
+    libpq-dev \
+    build-essential
+
+COPY ${FROM_DIR}/requirements.txt /
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+COPY ${FROM_DIR}/.creds  /.creds
+COPY ${FROM_DIR}/run_md_recorder.sh /run_md_recorder.sh
+
+
+# Shared Volumes
+RUN mkdir -p /app/data
+RUN mkdir /logs
+RUN chmod +x /run_md_recorder.sh
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENV PYTHONPATH=/
+ENTRYPOINT [ "/run_md_recorder.sh" ]
+

market_data/md_recorder/run_md_recorder.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# runs in container
+
+ValidJobs=('BNBFUT_CLD21' 'BNBSPOT_CLD21' 'COINBASE_CLD21' 'BNBFUT_CVTTDATA' 'BNBSPOT_CVTTDATA' 'COINBASE_CVTTDATA')
+# runs on host to start container
+usage() {
+  echo "Usage: $0 <job_name; one of (${ValidJobs[@]})>"
+  exit 1
+}
+
+is_valid() {
+  local job=$1
+  for valid_job in "${ValidJobs[@]}";
+  do
+    # echo "job=$job valid_job=$valid_job"
+    if [[ "${job}" == "${valid_job}" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+job=${1}
+if  ! is_valid "${job}"; then
+    usage
+fi
+
+# ConfigServer=cloud16.cvtt.vpn
+ConfigServer=cloud23.cvtt.vpn:6789
+
+Cmd="python3.12"
+Cmd="${Cmd} cvttpy/apps/md/md_recorder.py"
+Cmd="${Cmd} --config=http://${ConfigServer}/apps/md_recorder"
+Cmd="${Cmd} --credentials_file=/.creds"
+Cmd="${Cmd} --compress_log"
+Cmd="${Cmd} --instrument_group=${job}"
+Cmd="${Cmd} --log_file=/logs/%T.MD_REC.${job}.log"
+if [ "${job}" == "BNBFUT_CLD21" ] ; then
+    Cmd="${Cmd} --active_exchanges=BNBFUT"
+    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
+    Cmd="${Cmd} --admin_port=7201"
+elif [ "${job}" == "COINBASE_CLD21" ] ; then
+    Cmd="${Cmd} --active_exchanges=COINBASE_AT"
+    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
+    Cmd="${Cmd} --admin_port=7202"
+elif [ "${job}" == "BNBSPOT_CLD21" ] ; then
+    Cmd="${Cmd} --active_exchanges=BNBSPOT"
+    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
+    Cmd="${Cmd} --admin_port=7203"
+elif [ "${job}" == "BNBSPOT_CVTTDATA" ] ; then
+    Cmd="${Cmd} --active_exchanges=BNBSPOT"
+    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
+    Cmd="${Cmd} --admin_port=7204"
+elif [ "${job}" == "BNBFUT_CVTTDATA" ] ; then
+    Cmd="${Cmd} --active_exchanges=BNBFUT"
+    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
+    Cmd="${Cmd} --admin_port=7205"
+elif [ "${job}" == "COINBASE_CVTTDATA" ] ; then
+    Cmd="${Cmd} --active_exchanges=COINBASE_AT"
+    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
+    Cmd="${Cmd} --admin_port=7206"
+else
+    echo "Unrecognized JOB: ${job}"
+    exit 1
+fi
+echo ${Cmd}
+eval ${Cmd}
+
+

market_data/md_recorder_monitor/.dockerignore

@@ -0,0 +1,3 @@
+.git
+**/__pycache__
+.pipenv

market_data/md_recorder_monitor/Dockerfile

@@ -13,7 +13,7 @@ RUN pip install --upgrade pip --root-user-action=ignore
 RUN pip install -r /requirements.txt --root-user-action=ignore
 
 COPY cvttpy /cvttpy
-# COPY ${FROM_DIR}/.creds  /.creds
+COPY ${FROM_DIR}/.creds  /.creds
 COPY ${FROM_DIR}/run_mdr_monitor.sh /run_mdr_monitor.sh
 
 

market_data/md_recorder_monitor/requirements.txt

@@ -0,0 +1,7 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5
+psycopg>=3.2.1
+hjson>=3.1.0
+pandas>=1.5.3
+sortedcontainers>=2.4.0
+redis>=5.0.8

market_data/md_recorder_monitor/run_mdr_monitor.sh

@@ -32,7 +32,8 @@ if [ "${2}" != "" ]; then
 fi
 
 
-ConfigServer=cloud16.cvtt.vpn:6789
+# ConfigServer=cloud16.cvtt.vpn
+ConfigServer=cloud23.cvtt.vpn:6789
 
 Cmd="python3.12"
 Cmd="${Cmd} cvttpy/apps/md/md_rec_monitor.py"

market_data/relative_liquidity/.creds

@@ -1,10 +0,0 @@
-{
-    "__dummy": null
-    , "INFISICAL": {
-        "url": "https://infisical.cvtt.net"
-        , "client_id": "e330a1c4-08dc-452f-ac02-5b37fff4ab3e"
-        , "client_secret": "190554828c3c342f63517320ac029f822b6358aa6187aec266465d0bde4d6b3c"
-        , "workspace_id": "6923c950-8473-4cec-818f-10df41d8b39c"
-        , "environment": "dev"
-    }
-}

market_data/relative_liquidity/Dockerfile

@@ -1,10 +1,7 @@
 FROM python:3.12-slim
-ARG FROM_DIR=docker_dev/market_data/relative_liquidity
 
 WORKDIR /
-COPY ${FROM_DIR}/.creds  /.creds
-COPY ${FROM_DIR}/requirements.txt /requirements.txt
-
+COPY docker_dev/relative_liquidity/requirements.txt /
 RUN pip install --upgrade pip --root-user-action=ignore
 RUN pip install -r /requirements.txt --root-user-action=ignore
 
@@ -15,12 +12,22 @@ RUN mkdir -p /app/data
 RUN mkdir /logs
 
 ENV PYTHONPATH=/
-CMD [                                                            \
-    "python3.12",                                                \
-    "cvttpy/apps/md/rel_lqdt_server.py",                         \
-    "--config=http://cloud16.cvtt.vpn:6789/apps/minimal_md_eqt", \
-    "--credentials_file=/.creds",                                \
-    "--port=5678",                                               \
+CMD [                                        \
+    "python3.12",                            \
+    "cvttpy/apps/md/rel_lqdt_server.py",    \
+    "--port=5678",                           \
     "--db_file=/app/data/rel_liquidity.db",                      \
-    "--log_file=/logs/%D.REL_LIQUIDITY_SVC.log"                  \
+    "--log_file=/logs/%%T.REL_LIQUIDITY_SVC.log" \
 ]
+
+
+# Cmd="docker run"
+# Cmd="${Cmd} -d"
+# Cmd="${Cmd} --rm"
+# Cmd="${Cmd} --pull=always"
+# Cmd="${Cmd} --name=relative_liquidity_svc"
+# Cmd="${Cmd} -p 5678:5678"
+# Cmd="${Cmd} -v /home/cvtt/prod/data:/app/data"
+# Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
+# Cmd="${Cmd} cloud21.cvtt.vpn:5500/relative_liquidity:latest"
+

market_data/relative_liquidity/requirements.txt

@@ -3,4 +3,4 @@ nest-asyncio>=1.5.5
 numpy>=1.24.1
 hjson>=3.1.0
 pandas>=1.5.3
-sortedcontainers>=2.4.0
+# sortedcontainers>=2.4.0

release_version.txt

@@ -0,0 +1 @@
+0.0.3,alpaca_qat loader from disted version

research/crypto_exch_stats/Dockerfile

@@ -1,31 +0,0 @@
-FROM python:3.12-slim
-
-ARG FROM_DIR=./docker_dev/research/crypto_exch_stats
-
-WORKDIR /
-COPY ${FROM_DIR}/requirements.txt /
-
-RUN pip install --upgrade pip --root-user-action=ignore
-RUN pip install -r /requirements.txt --root-user-action=ignore
-
-COPY cvttpy /cvttpy
-
-# Shared Volumes
-RUN mkdir -p /app/data
-RUN mkdir /logs
-
-ENV PYTHONPATH=/
-CMD [                                                                      \
-    "python3.12",                                                          \
-    "cvttpy/apps//research/exchange_trading_stats.py",                     \
-    "--config=http://cloud16.cvtt.vpn:6789/apps/tests/listen_market_data", \
-    "--active_exchanges=OKX,GEMINI,BITSTAMP,COINBASE_AT,BNBSPOT,KRAKEN",   \
-    "--instruments=OKX:PAIR-BTC-USDT,GEMINI:PAIR-BTC-USD,BITSTAMP:PAIR-BTC-USD,COINBASE:PAIR-BTC-USD,BNBSPOT:PAIR-BTC-USDT,KRAKEN:PAIR-BTC-USD", \
-    "--db_file=/app/data/exchange_trading_stats.db",                       \
-    "--log_level=INFO",                                                    \
-    "--log_file=/logs/%T.EXCHANGE_TRADING_STATS.log"                       \
-]
-
-# ---------- No BNBSPOT from USA
-# "--active_exchanges=OKX,GEMINI,BITSTAMP,COINBASE_AT,KRAKEN",   \
-# "--instruments=OKX:PAIR-BTC-USDT,GEMINI:PAIR-BTC-USD,BITSTAMP:PAIR-BTC-USD,COINBASE:PAIR-BTC-USD,KRAKEN:PAIR-BTC-USD", \

research/crypto_exch_stats/run.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-Cmd="docker run"
-Cmd+=" -d"
-# Cmd+=" --rm"
-Cmd+=" --pull=always"
-Cmd+=" --network=host"
-Cmd+=" --name=crypto_exch_stats"
-Cmd+=" --volume=${HOME}/prod/data:/app/data"
-Cmd+=" --volume=${HOME}/prod/logs:/logs"
-Cmd+=" cloud21.cvtt.vpn:5500/crypto_exch_stats:latest"
-
-echo ${Cmd}
-eval ${Cmd}
-# Cmd+=" cloud21.cvtt.vpn:5500/relative_liquidity:latest"
-

shared/known_hosts

@@ -0,0 +1,6 @@
+|1|yT6WmkWOPxXSQeNECVJIX7M3upw=|wQPqbw6mXxwfO83sWVH9IRapSe0= ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNIOY8dNaWBcUu31F2OOuXUFGam2sx3Admn51SZR37MOMDEeKzpX+w0OnDGME9uOfuBToDW5F8ZoC+/p492uywZb4ZvUqVE8ic3uTaMro7zB9sykt4AO2rifM/0k7Fo1WoNWSaQptS3qyf7Umgl+qD5H8PxEM1OqKQl38BI5hfKUGx/WSYawgGIKkyO1eFwdjmSJ4RgQpN6/p5+1wJqbqbzkk0cEvZ06BbUbY+nUSUYY5WKa8ICBI0GKDdgRg9FXTV18r0YPT62HhShIyScbmg2h9hDKb7q9lLNLs2qg11jpSZCsIYTR2yms60uqXXq+isReLZ+W4z0t/8YVoID4Q82UlJ8R/LEqiJd8VcjZHaRVqtr7oVQjSurg1ChgCnOHHjvjZLVUg7QRMbCVT2/6PMAjjTyHUx8TKscYjz21ctPMn63pH/6lYzncQrt7ZT6PromVtnBs7p8dHaCyv18z9EBvGuE2rYOsfsCuR7FNlno7ej/gbe6CoaT2KwcNU8vpc=
+|1|QgYdyhwM1uy97fUH/l9jXfRnYlE=|P575Q9H/V0Hoszk+G9As1GozQmE= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHlFA8BawdNbgYsFPPUJiG7xfuWI6dwNIFkuUwqco+TPXanMG4CvZ0TjRgb2fwdMXAGyoNAmwcr8eR5HU8wTg8g=
+|1|vSzhghnKpUIGuOOgELHWGrbMxd8=|GY2Acsr6iqo4Y/dDS8V5glZ7tmA= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKz0QyrgFpp9NsPvZVwkjoua57SztFOMUnEzOLfW7kPr
+|1|LJg/h/d7KSfFYzdtrrEFpcpnF/4=|xRkJEoqiptYGTTt5EuQo4f3zo6c= ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKnC41A86Q2mVQWyRXHKzht6t8J1JJsRbYrQRpcMzh6VkVmOPDdAtl4P5iOH/GUaHr+HNhS7Y/QZS0yyjv8u1yOjb6ovfKvIuysj7f87wI1UOtQaUgX/PG5wupk/SgiUcfkpG+hg5wUACbsbMjp+HbE/OB7Nk5ZzUHNZcwQ55mRis8AzLW3IuFEboE+tJCxqTz8mr95ecqEWXdG4cJdDp7BMuCzsPCsOWgpcJiH+kgimok9n4qJPgoFSTVvHKBj0uJlm33rc8ns/Uu8knq6QfeSlzlN7uW0Xue6irW4Uaz69NbSUcFCeG83WNGO7tceYc7tmVM+9jjAtyhh9x1kGyk6thBMimRE5taa+5+bZP5tMctFsze6LVUGOothYW2MVgllJ+ovNFV/lin1y1591eAl340uSs4E4AJH6F/2Tta9OVGvNaXGMTYzKOLBYwo9sXJRHFuPjFxEs0Ya4pFUoCa8B53reNiOafrtwRU8/8G6zMgrF5z4FhCYBcIzoJdWCE=
+|1|9oemW7STLSSF7uf+SBwD7JWe0K8=|YCblOLOrxHLzbHkMuZfiUP8nkTU= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPEpfGoofmKWz3/t3Svaahh5ieq4LJ6/n+y3M0rOK/RgndUCESaTZXbw701NOF8BPcJczTQTuQPO2AJwrwQYcK0=
+|1|EC/isVVKdC1w/IthdEuhT29iU+w=|VClak/HfuZVXh4zlhhtx9euaTT0= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA2qvtcS1snKAdWKJefMCuKKgVlz0haOT0sElceM06qV

tools/code-server-plus/Dockerfile

@@ -1,49 +0,0 @@
-FROM codercom/code-server:latest
-
-ARG FROM_DIR=./
-USER root
-# Update the package list and install required packages
-RUN sudo apt-get update && apt-get install -y \
-    apt-utils \
-    rsync \
-    openssh-client \
-    software-properties-common \ 
-    build-essential \
-    zlib1g-dev \
-    libncurses5-dev \
-    libgdbm-dev \ 
-    libnss3-dev \
-    libssl-dev \
-    libreadline-dev \
-    libffi-dev \
-    libsqlite3-dev \
-    libbz2-dev \
-    curl \
-    wget \
-    jq
-
-# ----------------------------------------
-RUN mkdir -p /tmp
-WORKDIR /tmp
-RUN wget https://www.python.org/ftp/python/3.12.5/Python-3.12.5.tgz
-
-RUN tar -xf Python-3.12.5.tgz
-WORKDIR /tmp/Python-3.12.5
-
-RUN ./configure --enable-optimizations --prefix=/usr
-RUN make -j$(nproc)
-RUN make altinstall
-
-# ----------------------------------------
-RUN rm /usr/bin/python3
-RUN ln -s /usr/bin/python3.12 /usr/bin/python3
-
-RUN ln -snf /usr/bin/pip3.12 /usr/bin/pip
-
-COPY ./requirements.txt /
-RUN pip install --upgrade pip --root-user-action=ignore
-RUN pip install -r /requirements.txt --root-user-action=ignore
-
-SHELL ["/bin/bash", "-c"]
-
-WORKDIR /home/coder

tools/code-server-plus/build.sh

@@ -1,2 +0,0 @@
-#!/bin/bash
-docker build -t code-server-plus -t code-server-plus:latest -f Dockerfile .

tools/code-server-plus/docker-compose.yml

@@ -1,15 +0,0 @@
-version: "3.8"
-services:
-  code-server:
-    container_name: code-server-plus
-    image: code-server-plus:latest
-    restart: unless-stopped
-    user: 1001:1001
-    volumes:
-      - /opt/code-server/.local:/home/coder/.local
-      - /opt/code-server/.config:/home/coder/.config
-      - /opt/code-server/projects:/home/coder/project
-    environment:
-      - DOCKER_USER=cvtt
-    ports:
-      - "8080:8080"

tools/code-server-plus/requirements.txt

@@ -1,162 +0,0 @@
-aiohttp>=3.7.4.post0
-appdirs>=1.4.4
-asn1crypto>=0.24.0
-astroid>=2.8.0
-async-timeout>=3.0.1
-attrs>=22.1.0
-Automat>=0.8.0
-backports.entry-points-selectable>=1.1.0
-bcrypt>=4.0.0
-black>=22.6.0
-blinker>=1.4
-boto3>=1.24.43
-botocore>=1.27.43
-cachetools>=4.1.1
-certifi>=2019.11.28
-cffi>=1.15.0
-chardet>=4.0.0
-charset-normalizer>=2.0.7
-ciso8601>=2.2.0
-click>=8.1.3
-coloredlogs>=15.0.1
-configobj>=5.0.6
-constantly>=15.1.0
-cryptography>=2.8
-dataclasses-json>=0.5.1
-distlib>=0.3.3
-distro>=1.4.0
-docker>=5.0.3
-entrypoints>=0.3
-eventkit>=0.8.9
-ExportCsvToInflux>=0.2.2
-filelock>=3.3.2
-google-api-core>=1.22.2
-google-api-python-client>=2.22.0
-google-auth>=1.21.1
-google-auth-httplib2>=0.1.0
-google-auth-oauthlib>=0.4.6
-googleapis-common-protos>=1.52.0
-greenlet>=1.1.2
-hiredis>=1.1.0
-hjson>=3.0.2
-httplib2>=0.18.1
-humanfriendly>=10.0
-hyperlink>=19.0.0
-ib-insync>=0.9.70
-ibapi
-idna>=2.8
-importlab>=0.7
-importlib-metadata>=1.5.0
-incremental>=16.10.1
-influxdb>=5.3.1
-influxdb-client>=1.29.1
-isort>=5.5.1
-jedi>=0.18.0
-Jinja2>=2.10.1
-jmespath>=1.0.1
-json-cfg>=0.4.2
-jsonpatch>=1.22
-jsonpointer>=2.0
-jsonschema>=3.2.0
-keyring>=18.0.1
-kwonly-args>=1.0.10
-launchpadlib>=1.10.13
-lazr.restfulclient>=0.14.2
-lazr.uri>=1.0.3
-lazy-object-proxy>=1.4.3
-libcst>=0.4.7
-lxml>=4.6.3
-lxml-stubs>=0.2.0
-MarkupSafe>=1.1.0
-marshmallow>=3.7.0
-marshmallow-enum>=1.5.1
-mccabe>=0.6.1
-more-itertools>=4.2.0
-msgpack>=0.6.1
-multidict>=4.7.6
-multitasking>=0.0.10
-mypy>=0.961
-mypy-extensions>=0.4.3
-nest-asyncio>=1.5.5
-networkx>=2.8.3
-ninja>=1.10.2.3
-numpy #>=1.19.1
-oauthlib>=3.1.0
-pandas>=1.3.3
-pandas-stubs>=1.2.0.17
-paramiko>=2.11.0
-parso>=0.8.1
-pathspec>=0.9.0
-pbr>=5.6.0
-pexpect>=4.6.0
-platformdirs>=2.3.0
-protobuf>=3.13.0
-ptyprocess>=0.7.0
-pyasn1>=0.4.2
-pyasn1-modules>=0.2.1
-pycparser>=2.20
-pydot>=1.4.2
-PyHamcrest>=1.9.0
-PyJWT>=1.7.1
-pylint>=2.11.1
-pymongo>=4.0.1
-PyNaCl>=1.5.0
-pynvim>=0.4.1
-pyOpenSSL>=19.0.0
-pyparsing>=3.0.9
-pyrsistent>=0.15.5
-pyserial>=3.4
-python-dateutil>=2.8.2
-python-rapidjson>=0.9.1
-pytz>=2020.1
-PyYAML>=5.3.1
-redis>=5.0.8
-regex>=2021.11.10
-requests>=2.27.1
-requests-oauthlib>=1.3.0
-requests-unixsocket>=0.2.0
-rsa>=4.6
-Rx>=3.2.0
-s3transfer>=0.6.0
-SecretStorage>=2.3.1
-service-identity>=18.1.0
-simplefix>=1.0.14
-simplejson>=3.16.0
-six>=1.14.0
-slackclient>=2.9.3
-sortedcontainers>=2.4.0
-SQLAlchemy>=1.4.39
-ssh-import-id>=5.10
-stringcase>=1.2.0
-tabulate>=0.8.10
-termcolor>=1.1.0
-testresources>=2.0.1
-toml>=0.10.2
-tomli>=2.0.1
-typed-ast>=1.5.1
-types-chardet>=4.0.1
-types-cryptography>=3.3.23
-types-paramiko>=2.11.6
-types-PyMySQL>=1.0.18
-types-python-dateutil>=2.8.0
-types-pytz>=2022.1.1
-types-redis>=4.2.6
-types-requests>=2.25.7
-types-simplejson>=3.17.0
-types-termcolor>=1.1.1
-typing-inspect>=0.6.0
-typing_extensions>=4.3.0
-uritemplate>=3.0.1
-urllib3>=1.26.9
-uvloop>=0.14.0
-virtualenv>=20.9.0
-wadllib>=1.3.3
-websocket-client>=0.53.0
-websockets>=8.1
-wrapt>=1.12.1
-yapf>=0.30.0
-yarl>=1.5.1
-yfinance>=0.1.70
-zipp>=1.0.0
-zope.interface>=5.4.0
-

trading/executor/.creds

@@ -0,0 +1,12 @@
+{
+    "__dummy__": null
+    , "ALPACA_SANDBOX": {
+        "api_key": "PKLZSLFZMFMN1R28K9HK"
+        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
+    }
+    , "MATTERMOST": {
+        "url": "https://mattermost.cryptovaltrading.com"
+        , "team": "CVTT"
+        , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
+    }
+}

trading/executor/.dockerignore

@@ -0,0 +1,3 @@
+.git
+**/__pycache__
+.pipenv

trading/executor/Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.12-slim
+
+
+ARG FROM_DIR=docker_dev/trading/executor
+
+COPY ${FROM_DIR}/requirements.txt /
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+COPY ${FROM_DIR}/.creds  /.creds
+
+
+COPY ${FROM_DIR}/run_executor.sh /run_executor.sh
+RUN chmod +x /run_executor.sh
+
+RUN mkdir /logs
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENV PYTHONPATH=/
+ENTRYPOINT [ "/run_executor.sh" ]
+
+

trading/executor/requirements.txt

@@ -0,0 +1,6 @@
+aiohttp>=3.7.4.post0
+redis>=5.0.8
+nest-asyncio>=1.5.5
+hjson>=3.1.0
+sortedcontainers>=2.4.0
+pandas>=1.5.3

trading/executor/run_executor.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# runs in container
+
+
+# runs on host to start container
+usage() {
+    echo -n "Usage: $0"
+    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
+    echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+    echo -n " [-a <admin_port (7222)>"]
+    echo -n " [-z (compress log)"]
+    echo
+    exit 1
+}
+
+Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
+ActiveExchanges=ALPACA_SNDBX
+AdminPort=7222
+COMPRESS_LOG=false
+
+while getopts "c:e:a:z" opt; do
+    case ${opt} in
+        c ) Config=$OPTARG  ;;
+        e ) ActiveExchanges=$OPTARG ;;
+        a ) AdminPort=$OPTARG ;;
+        z ) COMPRESS_LOG=true ;;
+        \? )
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            ;;
+        : )
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            ;;
+    esac
+done
+
+Cmd="python3.12"
+Cmd="${Cmd} /cvttpy/apps/executor_app.py"
+Cmd="${Cmd} --config=${Config}"
+Cmd="${Cmd} --credentials_file=/.creds"
+Cmd="${Cmd} --allow_dynamic_exch_inst"
+Cmd="${Cmd} --active_exchanges=${ActiveExchanges}"
+Cmd="${Cmd} --admin_port=${AdminPort}"
+Cmd="${Cmd} --log_file=/logs/%T.EXECUTOR_APP.log"
+if ${COMPRESS_LOG}; then
+    Cmd="${Cmd} --compress_log"
+fi
+echo ${Cmd}
+eval ${Cmd}
+
+

trading/md_portal/.creds

@@ -0,0 +1,7 @@
+{
+    "__dummy__": null
+    , "ALPACA_SANDBOX": {
+        "api_key": "PKLZSLFZMFMN1R28K9HK"
+        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
+    }
+}

trading/md_portal/.dockerignore

@@ -0,0 +1,3 @@
+.git
+**/__pycache__
+.pipenv

trading/md_portal/Dockerfile

@@ -0,0 +1,23 @@
+FROM python:3.12-slim
+
+ARG FROM_DIR=docker_dev/trading/md_portal
+
+COPY ${FROM_DIR}/requirements.txt /
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+COPY ${FROM_DIR}/.creds  /.creds
+
+
+COPY ${FROM_DIR}/run_md_portal.sh /run_md_portal.sh
+RUN chmod +x /run_md_portal.sh
+
+RUN mkdir /logs
+
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENV PYTHONPATH=/
+ENTRYPOINT [ "/run_md_portal.sh" ]
+

trading/md_portal/requirements.txt

@@ -0,0 +1,7 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5
+hjson>=3.1.0
+sortedcontainers>=2.4.0
+redis>=5.0.8
+python-dateutil>=2.8.2
+types-python-dateutil>=2.8.19.6

trading/md_portal/run_md_portal.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# runs in container
+
+# runs on host to start container
+usage() {
+    echo -n "Usage: ${0}"
+    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
+    echo -n " [-e <active_exchanges (ALPACA_SNBOX)>]"
+    echo -n " [-a <admin_port (7220)>]"
+    echo -n " [-n <portal_name (MD_PORTAL_ALPACA)>]"
+    echo -n " [-z (compress log)"]
+    echo
+    exit 1
+}
+
+Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
+ActiveExchanges=ALPACA_SNDBX
+PortalName=MD_PORTAL_ALPACA_SNDBX
+AdminPort=7220
+COMPRESS_LOG=false
+
+
+while getopts "c:e:a:n:z" opt; do
+    case ${opt} in
+        c ) Config=$OPTARG  ;;
+        e ) ActiveExchanges=$OPTARG ;;
+        a ) AdminPort=$OPTARG ;;
+        z ) COMPRESS_LOG=true ;;
+        n ) PortalName=$OPTARG ;;
+        \? )
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            ;;
+        : )
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            ;;
+    esac
+done
+
+Cmd="python3.12"
+Cmd="${Cmd} cvttpy/apps/md/md_portal.py"
+Cmd="${Cmd} --config=${Config}"
+Cmd="${Cmd} --credentials_file=/.creds"
+Cmd="${Cmd} --active_exchanges=${ActiveExchanges}"
+Cmd="${Cmd} --portal_name=${PortalName}"
+Cmd="${Cmd} --admin_port=${AdminPort}"
+Cmd="${Cmd} --log_file=/logs/%T.${PortalName}.log"
+if ${COMPRESS_LOG}; then
+    Cmd="${Cmd} --compress_log"
+fi
+echo ${Cmd}
+eval ${Cmd}
+
+

trading/quant/.creds

@@ -0,0 +1,21 @@
+{
+    "TSDB_MD_CLD19_RO": {
+        "host": "cloud19.cvtt.vpn"
+        , "port": 5432
+        , "user": "cvtt_ro"
+        , "database": "cvtt_md"
+        , "password": "3Gkotqa0ZSmZLXSH3RkTUm"
+    }
+    , "TSDB_MD_CVTTDATA_RO": {
+        "host": "cvttdata.cvtt.vpn"
+        , "port": 5432
+        , "user": "cvtt_ro"
+        , "database": "cvtt_md"
+        , "password": "3Gkotqa0ZSmZLXSH3RkTUm"
+    }
+    , "MATTERMOST": {
+        "url": "https://cloud11.cryptovaltrading.com"
+        , "team": "CVTT"
+        , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
+    }
+}

trading/quant/.dockerignore

@@ -0,0 +1,3 @@
+.git
+**/__pycache__
+.pipenv

trading/quant/Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.12-slim
+
+
+ARG FROM_DIR=docker_dev/trading/quant
+
+COPY ${FROM_DIR}/requirements.txt /
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+COPY ${FROM_DIR}/.creds  /.creds
+
+
+COPY ${FROM_DIR}/run_quant.sh /run_quant.sh
+RUN chmod +x /run_quant.sh
+
+RUN mkdir /logs
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENV PYTHONPATH=/
+ENTRYPOINT [ "/run_quant.sh" ]
+
+

trading/quant/requirements.txt

@@ -0,0 +1,11 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5
+hjson>=3.1.0
+sortedcontainers>=2.4.0
+pandas>=1.5.3
+#
+ta>=0.10.2
+tables>=3.9.1
+psycopg>=3.2.1
+#
+redis>=5.0.8

trading/quant/run_quant.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# runs on host to start container
+usage() {
+    echo -n "Usage: $0"
+    echo -n " -b <book>"
+
+    echo -n " -S <strategy> (dflt: TRDALGO_001)"
+
+    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
+    echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+    echo -n " [-a <admin_port (7224)>"]
+    echo -n " [-z (compress log)"]
+    echo
+    exit 1
+}
+
+Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
+ActiveExchanges=ALPACA_SNDBX-MDPORTAL
+AdminPort=7224
+Strategy=DAILY_STOCK_001
+Book=""
+COMPRESS_LOG=false
+
+
+while getopts "b:c:e:a:S:z" opt; do
+    case ${opt} in
+        c ) Config=$OPTARG  ;;
+        e ) ActiveExchanges=$OPTARG ;;
+        a ) AdminPort=$OPTARG ;;
+        z ) COMPRESS_LOG=true ;;
+        b ) Book=$OPTARG ;;
+        S ) Strategy=$OPTARG ;;
+        \? )
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            ;;
+        : )
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            ;;
+    esac
+done
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+Cmd="python3.12"
+Cmd="${Cmd} cvttpy/apps/quant_app.py"
+Cmd="${Cmd} --config=${Config}"
+Cmd="${Cmd} --credentials_file=/.creds"
+Cmd="${Cmd} --book=${Book}"
+Cmd="${Cmd} --strategy=${Strategy}"
+Cmd="${Cmd} --allow_dynamic_exch_inst"
+Cmd="${Cmd} --active_exchanges=${ActiveExchanges}"
+Cmd="${Cmd} --admin_port=${AdminPort}"
+Cmd="${Cmd} --log_file=/logs/%T.QUANT.${Book}.log"
+if ${COMPRESS_LOG}; then
+    Cmd="${Cmd} --compress_log"
+fi
+echo ${Cmd}
+eval ${Cmd}
+
+

trading/risk_mgr/.creds

@@ -0,0 +1,20 @@
+{
+    "__dummy__": null
+    , "ALPACA_SANDBOX": {
+        "api_key": "PKLZSLFZMFMN1R28K9HK"
+        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
+    }
+    , "TSDB_TRD_CVTTDATA": {
+        "host": "cvttdata.cvtt.vpn"
+        , "port": "5432"
+        , "database": "cvtt_trading"
+        , "user": "cvtt"
+        , "password": "ICdIh0JnMM7vM7Pf"
+    }
+    , "COINBASE_ADV_RO": {
+        "api_key": "Ip4QpRtRHBmWNpc3"
+        , "secret_key": "N6ZEMdIdUYUldkJRTUcCRKUFot4Fp2j0"
+        , "account_uuid": "???"
+        , "book_id": "CVTT_BK01"
+   }
+}

trading/risk_mgr/.dockerignore

@@ -0,0 +1,3 @@
+.git
+**/__pycache__
+.pipenv

trading/risk_mgr/Dockerfile

@@ -0,0 +1,23 @@
+FROM python:3.12-slim
+
+ARG FROM_DIR=docker_dev/trading/risk_mgr
+
+COPY ${FROM_DIR}/requirements.txt /
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+COPY ${FROM_DIR}/.creds  /.creds
+
+
+COPY ${FROM_DIR}/run_risk_mgr.sh /run_risk_mgr.sh
+RUN chmod +x /run_risk_mgr.sh
+
+RUN mkdir /logs
+
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENV PYTHONPATH=/
+ENTRYPOINT [ "/run_risk_mgr.sh" ]
+

trading/risk_mgr/requirements.txt

@@ -0,0 +1,15 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5
+hjson>=3.1.0
+sortedcontainers>=2.4.0
+pandas>=1.5.3
+#
+redis>=5.0.8
+#
+python-dateutil>=2.8.2
+types-python-dateutil>=2.8.19.6
+#
+psycopg>=3.2.1
+psycopg-binary>=3.2.1
+psycopg2-binary>=2.9.6
+

trading/risk_mgr/run_risk_mgr.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# runs in container
+
+# runs on host to start container
+usage() {
+    echo -n "Usage: $0"
+    echo -n " -b <book>"
+    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
+    echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+    echo -n " [-a <admin_port (7223)>"]
+    echo -n " [-z (compress log)"]
+    echo
+    exit 1
+}
+
+
+
+Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
+ActiveExchanges=ALPACA_SNDBX-MDPORTAL
+AdminPort=7223
+Book=""
+COMPRESS_LOG=false
+
+
+while getopts "b:c:e:a:z" opt; do
+    case ${opt} in
+        c ) Config=$OPTARG  ;;
+        e ) ActiveExchanges=$OPTARG ;;
+        a ) AdminPort=$OPTARG ;;
+        z ) COMPRESS_LOG=true ;;
+        b ) Book=$OPTARG ;;
+        \? )
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            ;;
+        : )
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            ;;
+    esac
+done
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+Cmd="python3.12"
+Cmd="${Cmd} cvttpy/apps/risk_manager_app.py"
+Cmd="${Cmd} --config=${Config}"
+Cmd="${Cmd} --credentials_file=/.creds"
+Cmd="${Cmd} --book=${Book}"
+Cmd="${Cmd} --allow_dynamic_exch_inst"
+Cmd="${Cmd} --active_exchanges=${ActiveExchanges}"
+Cmd="${Cmd} --admin_port=${AdminPort}"
+Cmd="${Cmd} --log_file=/logs/%T.RISK_MGR.${Book}.log"
+if ${COMPRESS_LOG}; then
+    Cmd="${Cmd} --compress_log"
+fi
+echo ${Cmd}
+eval ${Cmd}
+
+

trading/service_checker/.creds

@@ -0,0 +1,8 @@
+{
+    "__dummy__": null
+    , "MATTERMOST": {
+        "url": "https://mattermost.cryptovaltrading.com"
+        , "team": "CVTT"
+        , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
+    }
+}

trading/service_checker/Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3.12-slim
+
+ARG FROM_DIR=docker_dev/trading/service_checker
+
+RUN apt-get update && apt-get install -y telnet redis-tools
+
+
+COPY ${FROM_DIR}/requirements.txt /
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+COPY ${FROM_DIR}/.creds  /.creds
+
+
+COPY ${FROM_DIR}/run_svc_checker.sh /run_svc_checker.sh
+RUN chmod +x /run_svc_checker.sh
+
+# RUN mkdir /logs
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENV PYTHONPATH=/
+ENTRYPOINT [ "/run_svc_checker.sh" ]
+# ENTRYPOINT [ "/usr/bin/bash" ]
+

trading/service_checker/requirements.txt

@@ -0,0 +1,7 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5
+hjson>=3.1.0
+sortedcontainers>=2.4.0
+redis>=5.0.8
+python-dateutil>=2.8.2
+types-python-dateutil>=2.8.19.6

trading/service_checker/run_svc_checker.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# runs in container
+
+# runs on host to start container
+usage() {
+    echo -n "Usage: ${0}"
+    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
+    echo
+    exit 1
+}
+
+Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
+
+
+while getopts ":c:" opt; do
+    case ${opt} in
+        c )
+            Config=$OPTARG
+            ;;
+        \? )
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            ;;
+        : )
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            ;;
+    esac
+done
+
+
+Cmd="python3.12"
+Cmd="${Cmd} cvttpy/apps/utils/services_checker.py"
+Cmd="${Cmd} --config=${Config}"
+Cmd="${Cmd} --credentials_file=/.creds"
+Cmd="${Cmd} --log_level=ERROR"
+
+echo ${Cmd}
+eval ${Cmd}
+

trading/trader/.creds

@@ -0,0 +1,12 @@
+{
+    "__dummy__": null
+    , "ALPACA_SANDBOX": {
+        "api_key": "PKLZSLFZMFMN1R28K9HK"
+        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
+    }
+    , "MATTERMOST": {
+        "url": "https://mattermost.cryptovaltrading.com"
+        , "team": "CVTT"
+        , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
+    }
+}

trading/trader/.dockerignore

@@ -0,0 +1,3 @@
+.git
+**/__pycache__
+.pipenv

trading/trader/Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.12-slim
+
+
+ARG FROM_DIR=docker_dev/trading/trader
+
+COPY ${FROM_DIR}/requirements.txt /
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+COPY ${FROM_DIR}/.creds  /.creds
+
+
+COPY ${FROM_DIR}/run_trader.sh /run_trader.sh
+RUN chmod +x /run_trader.sh
+
+RUN mkdir /logs
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENV PYTHONPATH=/
+ENTRYPOINT [ "/run_trader.sh" ]
+
+

trading/trader/requirements.txt

@@ -0,0 +1,6 @@
+aiohttp>=3.7.4.post0
+redis>=5.0.8
+nest-asyncio>=1.5.5
+hjson>=3.1.0
+sortedcontainers>=2.4.0
+pandas>=1.5.3

trading/trader/run_trader.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# runs in container
+# --------------------------------------------------------------------------
+
+# runs on host to start container
+usage() {
+    echo -n "Usage: $0"
+    echo -n " -b <book>"
+
+    echo -n " -A <algo> (dflt: TRDALGO_001)"
+
+    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
+    echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+    echo -n " [-a <admin_port (7223)>"]
+    echo -n " [-z (compress log)"]
+    echo
+    exit 1
+}
+
+
+
+Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
+ActiveExchanges=ALPACA_SNDBX-MDPORTAL
+AdminPort=7226
+Algo=TRDALGO_001
+Book=""
+COMPRESS_LOG=false
+
+
+while getopts "b:c:e:a:A:z" opt; do
+    case ${opt} in
+        c ) Config=$OPTARG  ;;
+        e ) ActiveExchanges=$OPTARG ;;
+        a ) AdminPort=$OPTARG ;;
+        z ) COMPRESS_LOG=true ;;
+        b ) Book=$OPTARG ;;
+        A ) Algo=$OPTARG ;;
+        \? )
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            ;;
+        : )
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            ;;
+    esac
+done
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+Cmd="python3.12"
+Cmd="${Cmd} cvttpy/apps/trader_app.py"
+Cmd="${Cmd} --config=${Config}"
+Cmd="${Cmd} --credentials_file=/.creds"
+Cmd="${Cmd} --book=${Book}"
+Cmd="${Cmd} --algo=${Algo}"
+Cmd="${Cmd} --allow_dynamic_exch_inst"
+Cmd="${Cmd} --active_exchanges=${ActiveExchanges}"
+Cmd="${Cmd} --admin_port=${AdminPort}"
+Cmd="${Cmd} --log_file=/logs/%T.TRADER.${Book}.log"
+if ${COMPRESS_LOG}; then
+    Cmd="${Cmd} --compress_log"
+fi
+echo ${Cmd}
+eval ${Cmd}
+
+

trading/trading_recorder/.creds

@@ -0,0 +1,12 @@
+{
+    "__dummy__": null
+    , "ALPACA_SANDBOX": {
+        "api_key": "PKLZSLFZMFMN1R28K9HK"
+        , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
+    }
+    , "MATTERMOST": {
+        "url": "https://mattermost.cryptovaltrading.com"
+        , "team": "CVTT"
+        , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
+    }
+}

trading/trading_recorder/.dockerignore

@@ -0,0 +1,3 @@
+.git
+**/__pycache__
+.pipenv

trading/trading_recorder/Dockerfile

@@ -0,0 +1,23 @@
+FROM python:3.12-slim
+
+ARG FROM_DIR=docker_dev/trading/trading_recorder
+
+COPY ${FROM_DIR}/requirements.txt /
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+COPY cvttpy /cvttpy
+COPY ${FROM_DIR}/.creds  /.creds
+
+
+COPY ${FROM_DIR}/run_trading_recorder.sh /run_trading_recorder.sh
+RUN chmod +x /run_trading_recorder.sh
+
+RUN mkdir /logs
+
+
+WORKDIR /
+SHELL ["/bin/bash", "-c"]
+ENV PYTHONPATH=/
+ENTRYPOINT [ "/run_trading_recorder.sh" ]
+

trading/trading_recorder/requirements.txt

@@ -0,0 +1,15 @@
+aiohttp>=3.7.4.post0
+nest-asyncio>=1.5.5
+hjson>=3.1.0
+sortedcontainers>=2.4.0
+pandas>=1.5.3
+#
+redis>=5.0.8
+#
+python-dateutil>=2.8.2
+types-python-dateutil>=2.8.19.6
+#
+psycopg>=3.2.1
+psycopg-binary>=3.2.1
+psycopg2-binary>=2.9.6
+

trading/trading_recorder/run_trading_recorder.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+
+
+# runs on host to start container
+usage() {
+    echo -n "Usage: $0"
+    echo -n " -b <book>"
+    echo -n " [-c <config (dflt: http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca)>]"
+    echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+    echo -n " [-a <admin_port (7223)>"]
+    echo -n " [-z (compress log)"]
+    echo
+    exit 1
+}
+
+# ConfigServer=cloud16.cvtt.vpn
+ActiveExchanges=ALPACA_SNDBX-MDPORTAL
+AdminPort=7225
+Book=""
+Config=http://cloud23.cvtt.vpn:6789/apps/cvtt_eqt_alpaca
+COMPRESS_LOG=false
+
+
+while getopts "b:c:e:a:z" opt; do
+    case ${opt} in
+        c ) Config=$OPTARG  ;;
+        e ) ActiveExchanges=$OPTARG ;;
+        a ) AdminPort=$OPTARG ;;
+        z ) COMPRESS_LOG=true ;;
+        b ) Book=$OPTARG ;;
+        \? )
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            ;;
+        : )
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            ;;
+    esac
+done
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+Cmd="python3.12"
+Cmd="${Cmd} cvttpy/apps/trade_recorder.py"
+Cmd="${Cmd} --config=${Config}"
+Cmd="${Cmd} --credentials_file=/.creds"
+Cmd="${Cmd} --book=${Book}"
+Cmd="${Cmd} --allow_dynamic_exch_inst"
+Cmd="${Cmd} --active_exchanges=${ActiveExchanges}"
+Cmd="${Cmd} --admin_port=${AdminPort}"
+Cmd="${Cmd} --log_file=/logs/%T.TRADING_RECORDER.${Book}.log"
+if ${COMPRESS_LOG}; then
+    Cmd="${Cmd} --compress_log"
+fi
+echo ${Cmd}
+eval ${Cmd}