From d915a0dfd40f052df91756afa5e1eb85cfb7d4a0 Mon Sep 17 00:00:00 2001
From: Oleg Sheynin <osheynin@gmail.com>
Date: Fri, 9 Aug 2024 22:46:20 -0400
Subject: [PATCH] progress

---
 build.sh                                      | 11 ++-
 buildx.sh                                     | 67 +++++++++++++++++++
 market_data/alpaca_md_day/Dockerfile          |  9 ++-
 market_data/alpaca_md_day/requirements.txt    |  2 +-
 market_data/crypto_md_day/Dockerfile          | 11 ++-
 market_data/crypto_md_day/crypto_md_day.sh    |  3 +-
 market_data/crypto_md_day/requirements.txt    |  2 +-
 market_data/md_recorder/Dockerfile            |  4 ++
 market_data/md_recorder/requirements.txt      |  2 +-
 market_data/md_recorder_monitor/Dockerfile    |  5 ++
 .../md_recorder_monitor/requirements.txt      |  2 +-
 quant/Dockerfile                              |  5 ++
 quant/requirements.txt                        |  5 +-
 13 files changed, 113 insertions(+), 15 deletions(-)
 create mode 100755 buildx.sh

diff --git a/build.sh b/build.sh
index 56d6caf..2ee28e4 100755
--- a/build.sh
+++ b/build.sh
@@ -1,11 +1,12 @@
 #!/bin/bash
 
 function usage {
-    echo "Usage: ${0} <app_path (e.g. market_data/md_recorder_monitor)>"
+    echo "Usage: ${0} <app_path (e.g. market_data/md_recorder_monitor)> <version>"
     exit
 }
 
 AppPath=${1}
+CvttpyVersion=${2}
 
 if [ "${AppPath}" == "" ]
 then
@@ -27,7 +28,7 @@ Version=$(cat ${RootDir}/cvttpy/release_version.txt | awk -F, '{print $1}')
 
 cd ${RootDir}
 
-Cmd="docker build -t ${ImageName} -t ${ImageName}:${Version} -f ${DockerDir}/Dockerfile ."
+Cmd="docker build -t ${ImageName} -t ${ImageName}:latest -t ${ImageName}:${Version} -f ${DockerDir}/Dockerfile ."
 echo ${Cmd}
 eval ${Cmd}
 
@@ -35,7 +36,13 @@ Cmd="docker tag ${ImageName}:latest ${RegistryService}/${ImageName}:latest"
 echo ${Cmd}
 eval ${Cmd}
 
+Cmd="docker tag ${ImageName}:${Version} ${RegistryService}/${ImageName}:${Version}"
+echo ${Cmd}
+eval ${Cmd}
+
 Cmd="docker push ${RegistryService}/${ImageName}:latest"
 echo ${Cmd}
 eval ${Cmd}
 
+
+
diff --git a/buildx.sh b/buildx.sh
new file mode 100755
index 0000000..982481a
--- /dev/null
+++ b/buildx.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+
+function usage {
+    echo "Usage: ${0} <app_path (e.g. market_data/md_recorder_monitor)> <version>"
+    exit
+}
+
+AppPath=${1}
+CvttpyVersion=${2}
+
+if [ "${AppPath}" == "" ]
+then
+    usage
+fi
+
+# --- Settings
+DockerDir=$(realpath $(dirname ${0})/${AppPath})
+RootDir=$(realpath $(dirname ${0})/..)
+RegistryService=cloud21.cvtt.vpn:5500
+
+AppName=$(basename ${AppPath})
+
+ImageName=${AppName}
+ImageDir=${HOME}/docker_images
+mkdir -p ${ImageDir}
+
+Version=$(cat ${RootDir}/cvttpy/release_version.txt | awk -F, '{print $1}')
+
+cd ${RootDir}
+
+# Create and use a new builder instance
+Cmd="docker buildx create --use --name ${AppName}_builder"
+echo ${Cmd}
+eval ${Cmd}
+
+# Ensure buildx is bootstrapped
+Cmd="docker buildx inspect ${AppName}_builder --bootstrap"
+echo ${Cmd}
+eval ${Cmd}
+
+# Build the image using buildx, tagging with both 'latest' and the version number
+Cmd="docker buildx build --platform linux/amd64,linux/arm64 -t ${ImageName}:latest -t ${ImageName}:${Version} -f ${DockerDir}/Dockerfile . --push"
+echo ${Cmd}
+eval ${Cmd}
+
+# Tagging the images with the registry
+Cmd="docker tag ${ImageName}:latest ${RegistryService}/${ImageName}:latest"
+echo ${Cmd}
+eval ${Cmd}
+
+Cmd="docker tag ${ImageName}:${Version} ${RegistryService}/${ImageName}:${Version}"
+echo ${Cmd}
+eval ${Cmd}
+
+# Push the images to the registry
+Cmd="docker push ${RegistryService}/${ImageName}:latest"
+echo ${Cmd}
+eval ${Cmd}
+
+Cmd="docker push ${RegistryService}/${ImageName}:${Version}"
+echo ${Cmd}
+eval ${Cmd}
+
+# Optionally, remove the builder to clean up
+Cmd="docker buildx rm ${AppName}_builder"
+echo ${Cmd}
+eval ${Cmd}
diff --git a/market_data/alpaca_md_day/Dockerfile b/market_data/alpaca_md_day/Dockerfile
index c6c5523..86222c0 100644
--- a/market_data/alpaca_md_day/Dockerfile
+++ b/market_data/alpaca_md_day/Dockerfile
@@ -3,12 +3,15 @@ FROM python:3.10-slim
 ARG FROM_DIR=docker_dev/market_data/alpaca_md_day
 
 # Update the package list and install required packages
-RUN apt-get update && apt-get install -y --no-install-recommends \
+RUN apt-get update && apt-get install -y \
     apt-utils \
+    libpq-dev \
+    build-essential
+RUN apt-get update && apt-get install -y rsync openssh-client
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
     software-properties-common \
     curl \
-    rsync \
-    openssh-client \
     parallel \
     jq
 
diff --git a/market_data/alpaca_md_day/requirements.txt b/market_data/alpaca_md_day/requirements.txt
index 3045f17..d31c4a4 100644
--- a/market_data/alpaca_md_day/requirements.txt
+++ b/market_data/alpaca_md_day/requirements.txt
@@ -1,6 +1,6 @@
 aiohttp>=3.7.4.post0
 nest-asyncio>=1.5.5
-asyncpg>=0.27.0
+psycopg>=3.2.1
 hjson>=3.1.0
 pandas>=1.5.3
 sortedcontainers>=2.4.0
diff --git a/market_data/crypto_md_day/Dockerfile b/market_data/crypto_md_day/Dockerfile
index 5ae92c4..268f20a 100644
--- a/market_data/crypto_md_day/Dockerfile
+++ b/market_data/crypto_md_day/Dockerfile
@@ -3,14 +3,19 @@ FROM python:3.10-slim
 ARG FROM_DIR=docker_dev/market_data/crypto_md_day
 
 COPY ${FROM_DIR}/requirements.txt /
-RUN pip install --upgrade pip --root-user-action=ignore
-RUN pip install -r /requirements.txt --root-user-action=ignore
+
 
 COPY cvttpy /cvttpy
 COPY ${FROM_DIR}/.creds  /.creds
 COPY ${FROM_DIR}/crypto_md_day.sh /crypto_md_day.sh
 
-RUN apt-get update && apt-get install -y --no-install-recommends rsync openssh-client && rm -rf /var/lib/apt/lists/*
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+RUN apt-get update && apt-get install -y \
+    libpq-dev \
+    build-essential 
+RUN apt-get update && apt-get install -y rsync openssh-client
 
 COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
 COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
diff --git a/market_data/crypto_md_day/crypto_md_day.sh b/market_data/crypto_md_day/crypto_md_day.sh
index 3cc887b..f35ed62 100755
--- a/market_data/crypto_md_day/crypto_md_day.sh
+++ b/market_data/crypto_md_day/crypto_md_day.sh
@@ -8,7 +8,8 @@ usage() {
 
 DbHost=""
 Date=""
-Schemas=coinbase,bnbspot,bnbfut
+# Schemas=coinbase,bnbspot,bnbfut
+Schemas=coinbase,bnbspot
 
 while getopts ":h:d:s:" opt; do
   case ${opt} in
diff --git a/market_data/crypto_md_day/requirements.txt b/market_data/crypto_md_day/requirements.txt
index e09e76b..f628c17 100644
--- a/market_data/crypto_md_day/requirements.txt
+++ b/market_data/crypto_md_day/requirements.txt
@@ -1,6 +1,6 @@
 aiohttp>=3.7.4.post0
 nest-asyncio>=1.5.5
-asyncpg>=0.27.0
+psycopg>=3.2.1
 hjson>=3.1.0
 pandas>=1.5.3
 sortedcontainers>=2.4.0
\ No newline at end of file
diff --git a/market_data/md_recorder/Dockerfile b/market_data/md_recorder/Dockerfile
index 08c5d9c..a56857a 100644
--- a/market_data/md_recorder/Dockerfile
+++ b/market_data/md_recorder/Dockerfile
@@ -2,6 +2,10 @@ FROM python:3.10-slim
 
 ARG FROM_DIR=docker_dev/market_data/md_recorder
 
+RUN apt-get update && apt-get install -y \
+    apt-utils \
+    libpq-dev \
+    build-essential
 
 COPY ${FROM_DIR}/requirements.txt /
 RUN pip install --upgrade pip --root-user-action=ignore
diff --git a/market_data/md_recorder/requirements.txt b/market_data/md_recorder/requirements.txt
index 3045f17..d31c4a4 100644
--- a/market_data/md_recorder/requirements.txt
+++ b/market_data/md_recorder/requirements.txt
@@ -1,6 +1,6 @@
 aiohttp>=3.7.4.post0
 nest-asyncio>=1.5.5
-asyncpg>=0.27.0
+psycopg>=3.2.1
 hjson>=3.1.0
 pandas>=1.5.3
 sortedcontainers>=2.4.0
diff --git a/market_data/md_recorder_monitor/Dockerfile b/market_data/md_recorder_monitor/Dockerfile
index e8f14c5..05accdc 100644
--- a/market_data/md_recorder_monitor/Dockerfile
+++ b/market_data/md_recorder_monitor/Dockerfile
@@ -2,6 +2,11 @@ FROM python:3.10-slim
 
 ARG FROM_DIR=docker_dev/market_data/md_recorder_monitor
 
+RUN apt-get update && apt-get install -y \
+    apt-utils \
+    libpq-dev \
+    build-essential
+
 
 COPY ${FROM_DIR}/requirements.txt /
 RUN pip install --upgrade pip --root-user-action=ignore
diff --git a/market_data/md_recorder_monitor/requirements.txt b/market_data/md_recorder_monitor/requirements.txt
index 3045f17..d31c4a4 100644
--- a/market_data/md_recorder_monitor/requirements.txt
+++ b/market_data/md_recorder_monitor/requirements.txt
@@ -1,6 +1,6 @@
 aiohttp>=3.7.4.post0
 nest-asyncio>=1.5.5
-asyncpg>=0.27.0
+psycopg>=3.2.1
 hjson>=3.1.0
 pandas>=1.5.3
 sortedcontainers>=2.4.0
diff --git a/quant/Dockerfile b/quant/Dockerfile
index a6edaf3..cedc88b 100644
--- a/quant/Dockerfile
+++ b/quant/Dockerfile
@@ -6,6 +6,11 @@ COPY docker_dev/quant/requirements.txt /
 RUN pip install --upgrade pip
 RUN pip install -r /requirements.txt
 
+RUN apt-get update && apt-get install -y \
+    apt-utils \
+    libpq-dev \
+    build-essential
+
 
 COPY cvttpy /cvttpy
 COPY docker_dev/quant/.creds  /.creds
diff --git a/quant/requirements.txt b/quant/requirements.txt
index 02a8cd1..99f71b7 100644
--- a/quant/requirements.txt
+++ b/quant/requirements.txt
@@ -1,5 +1,4 @@
 aiohttp>=3.7.4.post0
-aioredis>=2.0.1
 nest-asyncio>=1.5.5
 hjson>=3.1.0
 sortedcontainers>=2.4.0
@@ -7,4 +6,6 @@ pandas>=1.5.3
 #
 ta>=0.10.2
 tables>=3.9.1
-asyncpg>=0.27.0
\ No newline at end of file
+psycopg>=3.2.1
+#
+aioredis>=2.0.1