progress

build.sh

@@ -1,11 +1,12 @@
 #!/bin/bash
 
 function usage {
-    echo "Usage: ${0} <app_path (e.g. market_data/md_recorder_monitor)>"
+    echo "Usage: ${0} <app_path (e.g. market_data/md_recorder_monitor)> <version>"
     exit
 }
 
 AppPath=${1}
+CvttpyVersion=${2}
 
 if [ "${AppPath}" == "" ]
 then
@@ -27,7 +28,7 @@ Version=$(cat ${RootDir}/cvttpy/release_version.txt | awk -F, '{print $1}')
 
 cd ${RootDir}
 
-Cmd="docker build -t ${ImageName} -t ${ImageName}:${Version} -f ${DockerDir}/Dockerfile ."
+Cmd="docker build -t ${ImageName} -t ${ImageName}:latest -t ${ImageName}:${Version} -f ${DockerDir}/Dockerfile ."
 echo ${Cmd}
 eval ${Cmd}
 
@@ -35,7 +36,13 @@ Cmd="docker tag ${ImageName}:latest ${RegistryService}/${ImageName}:latest"
 echo ${Cmd}
 eval ${Cmd}
 
+Cmd="docker tag ${ImageName}:${Version} ${RegistryService}/${ImageName}:${Version}"
+echo ${Cmd}
+eval ${Cmd}
+
 Cmd="docker push ${RegistryService}/${ImageName}:latest"
 echo ${Cmd}
 eval ${Cmd}
 
+
+

buildx.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+function usage {
+    echo "Usage: ${0} <app_path (e.g. market_data/md_recorder_monitor)> <version>"
+    exit
+}
+
+AppPath=${1}
+CvttpyVersion=${2}
+
+if [ "${AppPath}" == "" ]
+then
+    usage
+fi
+
+# --- Settings
+DockerDir=$(realpath $(dirname ${0})/${AppPath})
+RootDir=$(realpath $(dirname ${0})/..)
+RegistryService=cloud21.cvtt.vpn:5500
+
+AppName=$(basename ${AppPath})
+
+ImageName=${AppName}
+ImageDir=${HOME}/docker_images
+mkdir -p ${ImageDir}
+
+Version=$(cat ${RootDir}/cvttpy/release_version.txt | awk -F, '{print $1}')
+
+cd ${RootDir}
+
+# Create and use a new builder instance
+Cmd="docker buildx create --use --name ${AppName}_builder"
+echo ${Cmd}
+eval ${Cmd}
+
+# Ensure buildx is bootstrapped
+Cmd="docker buildx inspect ${AppName}_builder --bootstrap"
+echo ${Cmd}
+eval ${Cmd}
+
+# Build the image using buildx, tagging with both 'latest' and the version number
+Cmd="docker buildx build --platform linux/amd64,linux/arm64 -t ${ImageName}:latest -t ${ImageName}:${Version} -f ${DockerDir}/Dockerfile . --push"
+echo ${Cmd}
+eval ${Cmd}
+
+# Tagging the images with the registry
+Cmd="docker tag ${ImageName}:latest ${RegistryService}/${ImageName}:latest"
+echo ${Cmd}
+eval ${Cmd}
+
+Cmd="docker tag ${ImageName}:${Version} ${RegistryService}/${ImageName}:${Version}"
+echo ${Cmd}
+eval ${Cmd}
+
+# Push the images to the registry
+Cmd="docker push ${RegistryService}/${ImageName}:latest"
+echo ${Cmd}
+eval ${Cmd}
+
+Cmd="docker push ${RegistryService}/${ImageName}:${Version}"
+echo ${Cmd}
+eval ${Cmd}
+
+# Optionally, remove the builder to clean up
+Cmd="docker buildx rm ${AppName}_builder"
+echo ${Cmd}
+eval ${Cmd}

market_data/alpaca_md_day/Dockerfile

@@ -3,12 +3,15 @@ FROM python:3.10-slim
 ARG FROM_DIR=docker_dev/market_data/alpaca_md_day
 
 # Update the package list and install required packages
-RUN apt-get update && apt-get install -y --no-install-recommends \
+RUN apt-get update && apt-get install -y \
     apt-utils \
+    libpq-dev \
+    build-essential
+RUN apt-get update && apt-get install -y rsync openssh-client
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
     software-properties-common \
     curl \
-    rsync \
-    openssh-client \
     parallel \
     jq
 

market_data/alpaca_md_day/requirements.txt

@@ -1,6 +1,6 @@
 aiohttp>=3.7.4.post0
 nest-asyncio>=1.5.5
-asyncpg>=0.27.0
+psycopg>=3.2.1
 hjson>=3.1.0
 pandas>=1.5.3
 sortedcontainers>=2.4.0

market_data/crypto_md_day/Dockerfile

@@ -3,14 +3,19 @@ FROM python:3.10-slim
 ARG FROM_DIR=docker_dev/market_data/crypto_md_day
 
 COPY ${FROM_DIR}/requirements.txt /
-RUN pip install --upgrade pip --root-user-action=ignore
-RUN pip install -r /requirements.txt --root-user-action=ignore
+
 
 COPY cvttpy /cvttpy
 COPY ${FROM_DIR}/.creds  /.creds
 COPY ${FROM_DIR}/crypto_md_day.sh /crypto_md_day.sh
 
-RUN apt-get update && apt-get install -y --no-install-recommends rsync openssh-client && rm -rf /var/lib/apt/lists/*
+RUN pip install --upgrade pip --root-user-action=ignore
+RUN pip install -r /requirements.txt --root-user-action=ignore
+
+RUN apt-get update && apt-get install -y \
+    libpq-dev \
+    build-essential 
+RUN apt-get update && apt-get install -y rsync openssh-client
 
 COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
 COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub

market_data/crypto_md_day/crypto_md_day.sh

@@ -8,7 +8,8 @@ usage() {
 
 DbHost=""
 Date=""
-Schemas=coinbase,bnbspot,bnbfut
+# Schemas=coinbase,bnbspot,bnbfut
+Schemas=coinbase,bnbspot
 
 while getopts ":h:d:s:" opt; do
   case ${opt} in

market_data/crypto_md_day/requirements.txt

@@ -1,6 +1,6 @@
 aiohttp>=3.7.4.post0
 nest-asyncio>=1.5.5
-asyncpg>=0.27.0
+psycopg>=3.2.1
 hjson>=3.1.0
 pandas>=1.5.3
 sortedcontainers>=2.4.0

market_data/md_recorder/Dockerfile

@@ -2,6 +2,10 @@ FROM python:3.10-slim
 
 ARG FROM_DIR=docker_dev/market_data/md_recorder
 
+RUN apt-get update && apt-get install -y \
+    apt-utils \
+    libpq-dev \
+    build-essential
 
 COPY ${FROM_DIR}/requirements.txt /
 RUN pip install --upgrade pip --root-user-action=ignore

market_data/md_recorder/requirements.txt

@@ -1,6 +1,6 @@
 aiohttp>=3.7.4.post0
 nest-asyncio>=1.5.5
-asyncpg>=0.27.0
+psycopg>=3.2.1
 hjson>=3.1.0
 pandas>=1.5.3
 sortedcontainers>=2.4.0

market_data/md_recorder_monitor/Dockerfile

@@ -2,6 +2,11 @@ FROM python:3.10-slim
 
 ARG FROM_DIR=docker_dev/market_data/md_recorder_monitor
 
+RUN apt-get update && apt-get install -y \
+    apt-utils \
+    libpq-dev \
+    build-essential
+
 
 COPY ${FROM_DIR}/requirements.txt /
 RUN pip install --upgrade pip --root-user-action=ignore

market_data/md_recorder_monitor/requirements.txt

@@ -1,6 +1,6 @@
 aiohttp>=3.7.4.post0
 nest-asyncio>=1.5.5
-asyncpg>=0.27.0
+psycopg>=3.2.1
 hjson>=3.1.0
 pandas>=1.5.3
 sortedcontainers>=2.4.0

quant/Dockerfile

@@ -6,6 +6,11 @@ COPY docker_dev/quant/requirements.txt /
 RUN pip install --upgrade pip
 RUN pip install -r /requirements.txt
 
+RUN apt-get update && apt-get install -y \
+    apt-utils \
+    libpq-dev \
+    build-essential
+
 
 COPY cvttpy /cvttpy
 COPY docker_dev/quant/.creds  /.creds

quant/requirements.txt

@@ -1,5 +1,4 @@
 aiohttp>=3.7.4.post0
-aioredis>=2.0.1
 nest-asyncio>=1.5.5
 hjson>=3.1.0
 sortedcontainers>=2.4.0
@@ -7,4 +6,6 @@ pandas>=1.5.3
 #
 ta>=0.10.2
 tables>=3.9.1
-asyncpg>=0.27.0
+psycopg>=3.2.1
+#
+aioredis>=2.0.1