Compare commits

..

No commits in common. "master" and "v0.2.2" have entirely different histories.

38 changed files with 542 additions and 89 deletions

View File

@ -1 +1 @@
0.2.7,new md_recorder build 0.2.2,relative liquidity loads via http

22
config_service/Dockerfile Normal file
View File

@ -0,0 +1,22 @@
FROM python:3.12-slim
WORKDIR /
COPY docker_dev/config_service/requirements.txt /
RUN pip install --upgrade pip --root-user-action=ignore
RUN pip install -r /requirements.txt --root-user-action=ignore
COPY cvttpy /cvttpy
# Shared Volumes
RUN mkdir -p /app/data
RUN mkdir /logs
ENV PYTHONPATH=/
CMD [ \
"python3.12", \
"cvttpy/apps/utils/config_server.py", \
"--port=6789", \
"--root=/app/data", \
"--log_file=/logs/%T.config_service.log" \
]

42
config_service/HOWTO.md Normal file
View File

@ -0,0 +1,42 @@
# Build/Dev Host
## Build
```bash
TgtHostname=cloud16.cvtt.vpn
Version=$(cat cvttpy/VERSION)
cd /home/oleg/develop/cvtt2
docker build -t config_service -t config_service:${Version} -f cvttpy/utils/docker/config_service/Dockerfile .
## Deploy
docker save -o /tmp/cvtt_config_service.img.tar config_service
scp /tmp/cvtt_config_service.img.tar cvtt@${TgtHostname}:/tmp/
ssh cvtt@${TgtHostname}
```
## On target Machine make sure user can use docker
```bash
sudo usermod -a -G docker cvtt
```
**Re-login as cvtt after adding cvtt to the group**
```bash
docker rm -f cvtt_config_service
docker load -i /tmp/cvtt_config_service.img.tar
docker run -d --name=cvtt_config_service -p 6789:6789 -v /home/cvtt/prod/config_service/data:/app/data -v /home/cvtt/prod/logs:/logs config_service
```
## Restarting
```bash
docker restart cvtt_config_service
```
# Useful Commands
```bash
docker images
docker ps -a
docker image rm <image>
docker rm <container>
docker exec -it <container> /bin/bash
```

View File

@ -0,0 +1,2 @@
aiohttp>=3.7.4.post0
nest-asyncio>=1.5.5

View File

@ -1,76 +0,0 @@
#!/bin/bash
get_project_version() {
Project=${1}
Version=${2}
host="cloud21.cvtt.vpn"
port="22"
user="cvttdist"
rel_dir="/home/cvttdist/software/cvtt2"
if [ "${Version}" == "latest" ]; then
echo "Checking for latest version of ${Project} on ${user}@${host}:${rel_dir}"
Version=$(ssh -q -p ${port} ${user}@${host} "ls -tr ${rel_dir}/${Project} | tail -1" )
echo "Latest version is ${Version}"
fi
echo "Checking ${user}@${host} for ${rel_dir}/${Project}/${Version} ..."
if ssh -q -p ${port} ${user}@${host} "test -d ${rel_dir}/${Project}/${Version}"
then
echo "Getting Version \"${Version}\"..."
rsync_cmd="rsync -ahv -e \"ssh -p ${port}\""
rsync_cmd="${rsync_cmd} ${user}@${host}:${rel_dir}/${Project}/${Version}/"
rsync_cmd="${rsync_cmd} ./"
echo ${rsync_cmd}
eval ${rsync_cmd}
status=$?
if [ ${status} -eq 0 ]
then
echo "Loading successful..."
return 0
else
return 1
fi
else
echo "Not Found ${rel_dir}/${Project}/${Version} on ${user}@${host}"
return 1
fi
}
build_docker_image() {
ImageName=${1}
RegistryService=${2}
Version=${3}
Project=${4}
ProjectVersion=${5}
Cmd="docker build"
if [ "${ProjectVersion}" != "" ]; then
Cmd+=" --label ${Project}=\"${ProjectVersion}\""
fi
Cmd+=" -t ${ImageName}"
Cmd+=" -t ${ImageName}:latest"
Cmd+=" -t ${ImageName}:${Version}"
Cmd+=" -f Dockerfile"
Cmd+=" ."
echo ${Cmd}
eval ${Cmd} || exit
Cmd="docker tag ${ImageName}:latest ${RegistryService}/${ImageName}:latest"
echo ${Cmd}
eval ${Cmd} || exit
Cmd="docker tag ${ImageName}:${Version} ${RegistryService}/${ImageName}:${Version}"
echo ${Cmd}
eval ${Cmd} || exit
Cmd="docker push ${RegistryService}/${ImageName}:latest"
echo ${Cmd}
eval ${Cmd} || exit
Cmd="docker push ${RegistryService}/${ImageName}:${Version}"
echo ${Cmd}
eval ${Cmd} || exit
}

View File

@ -24,6 +24,7 @@ COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub
# Shared Volumes # Shared Volumes
RUN mkdir -p /data RUN mkdir -p /data
RUN mkdir /logs RUN mkdir /logs

View File

@ -6,6 +6,7 @@ COPY ${FROM_DIR}/requirements.txt /
COPY cvttpy /cvttpy COPY cvttpy /cvttpy
# COPY ${FROM_DIR}/.creds /.creds
COPY ${FROM_DIR}/crypto_md_day.sh /crypto_md_day.sh COPY ${FROM_DIR}/crypto_md_day.sh /crypto_md_day.sh
RUN pip install --upgrade pip --root-user-action=ignore RUN pip install --upgrade pip --root-user-action=ignore

View File

@ -39,8 +39,8 @@ if [ -z "${DbHost}" ] || [ -z "$Date" ]; then
fi fi
CredKey="" CredKey=""
if [ "${DbHost}" == "cloud28" ] ; then if [ "${DbHost}" == "cvttdata" ] ; then
CredKey=TSDB_MD_CLOUD28_RO CredKey=TSDB_MD_CVTTDATA_RO
elif [ "${DbHost}" == "cloud21" ] ; then elif [ "${DbHost}" == "cloud21" ] ; then
CredKey=TSDB_MD_CLD21_RO CredKey=TSDB_MD_CLD21_RO
else else
@ -51,16 +51,16 @@ fi
export PYTHONPATH=/ export PYTHONPATH=/
Cmd="python3.12" Cmd="python3.12"
Cmd+=" cvttpy/research/utils/archive_ts_md.py" Cmd="${Cmd} cvttpy/research/utils/archive_ts_md.py"
Cmd+=" --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder" Cmd="${Cmd} --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder"
Cmd+=" --db_credentials_key=${CredKey}" Cmd="${Cmd} --db_credentials_key=${CredKey}"
Cmd+=" --credentials_file=/.creds" Cmd="${Cmd} --credentials_file=/.creds"
Cmd+=" --date=${Date}" Cmd="${Cmd} --date=${Date}"
Cmd+=" --schemas=${Schemas}" Cmd="${Cmd} --schemas=${Schemas}"
Cmd+=" --root_dir=/app/data/${DbHost}" Cmd="${Cmd} --root_dir=/app/data/${DbHost}"
Cmd+=" --format=SQLite" Cmd="${Cmd} --format=SQLite"
Cmd+=" --compress" Cmd="${Cmd} --compress"
# Cmd+=" --log_file=/logs/%T.crypto_md_day.log" # Cmd="${Cmd} --log_file=/logs/%T.crypto_md_day.log"
echo ${Cmd} echo ${Cmd}
eval ${Cmd} eval ${Cmd}

View File

@ -0,0 +1,17 @@
{
"__dummy__": null
, "TSDB_MD_CLD21": {
"host": "cloud21.cvtt.vpn"
, "port": 5432
, "user": "cvtt"
, "database": "cvtt_md"
, "password": "ICdIh0JnMM7vM7Pf"
}
, "TSDB_MD_CVTTDATA": {
"host": "cvttdata.cvtt.vpn"
, "port": 5432
, "user": "cvtt"
, "database": "cvtt_md"
, "password": "ICdIh0JnMM7vM7Pf"
}
}

View File

@ -0,0 +1,3 @@
.git
**/__pycache__
.pipenv

View File

@ -0,0 +1,27 @@
FROM python:3.12-slim
ARG FROM_DIR=docker_dev/market_data/md_recorder
RUN apt-get update && apt-get install -y \
apt-utils \
libpq-dev \
build-essential
COPY ${FROM_DIR}/requirements.txt /
RUN pip install --upgrade pip --root-user-action=ignore
RUN pip install -r /requirements.txt --root-user-action=ignore
COPY cvttpy /cvttpy
COPY ${FROM_DIR}/run_md_recorder.sh /run_md_recorder.sh
# Shared Volumes
RUN mkdir -p /app/data
RUN mkdir /logs
RUN chmod +x /run_md_recorder.sh
WORKDIR /
SHELL ["/bin/bash", "-c"]
ENV PYTHONPATH=/
ENTRYPOINT [ "/run_md_recorder.sh" ]

View File

@ -0,0 +1,69 @@
#!/bin/bash
# runs in container
ValidJobs=('BNBFUT_CLD21' 'BNBSPOT_CLD21' 'COINBASE_CLD21' 'BNBFUT_CVTTDATA' 'BNBSPOT_CVTTDATA' 'COINBASE_CVTTDATA')
# runs on host to start container
usage() {
echo "Usage: $0 <job_name; one of (${ValidJobs[@]})>"
exit 1
}
is_valid() {
local job=$1
for valid_job in "${ValidJobs[@]}";
do
# echo "job=$job valid_job=$valid_job"
if [[ "${job}" == "${valid_job}" ]]; then
return 0
fi
done
return 1
}
job=${1}
if ! is_valid "${job}"; then
usage
fi
ConfigServer=cloud16.cvtt.vpn:6789
Cmd="python3.12"
Cmd="${Cmd} cvttpy/apps/md/md_recorder.py"
Cmd="${Cmd} --config=http://${ConfigServer}/apps/md_recorder"
Cmd="${Cmd} --credentials_file=/.creds"
Cmd="${Cmd} --compress_log"
Cmd="${Cmd} --instrument_group=${job}"
Cmd="${Cmd} --log_file=/logs/%T.MD_REC.${job}.log"
if [ "${job}" == "BNBFUT_CLD21" ] ; then
Cmd="${Cmd} --active_exchanges=BNBFUT"
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
Cmd="${Cmd} --admin_port=7201"
elif [ "${job}" == "COINBASE_CLD21" ] ; then
Cmd="${Cmd} --active_exchanges=COINBASE_AT"
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
Cmd="${Cmd} --admin_port=7202"
elif [ "${job}" == "BNBSPOT_CLD21" ] ; then
Cmd="${Cmd} --active_exchanges=BNBSPOT"
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
Cmd="${Cmd} --admin_port=7203"
elif [ "${job}" == "BNBSPOT_CVTTDATA" ] ; then
Cmd="${Cmd} --active_exchanges=BNBSPOT"
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
Cmd="${Cmd} --admin_port=7204"
elif [ "${job}" == "BNBFUT_CVTTDATA" ] ; then
Cmd="${Cmd} --active_exchanges=BNBFUT"
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
Cmd="${Cmd} --admin_port=7205"
elif [ "${job}" == "COINBASE_CVTTDATA" ] ; then
Cmd="${Cmd} --active_exchanges=COINBASE_AT"
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
Cmd="${Cmd} --admin_port=7206"
else
echo "Unrecognized JOB: ${job}"
exit 1
fi
echo ${Cmd}
eval ${Cmd}

View File

@ -0,0 +1,3 @@
.git
**/__pycache__
.pipenv

View File

@ -0,0 +1,7 @@
aiohttp>=3.7.4.post0
nest-asyncio>=1.5.5
psycopg>=3.2.1
hjson>=3.1.0
pandas>=1.5.3
sortedcontainers>=2.4.0
redis>=5.0.8

View File

@ -0,0 +1,52 @@
version: "3.9"
services:
md_gateway:
image: cloud21.cvtt.vpn:5500/md_gateway:latest
container_name: md_gateway
environment:
- REDIS_HOST=redis
volumes:
- ./.creds:/.creds
- ./config:/config
- ./logs:/logs
- ./data:/data
depends_on:
- redis
tester:
image: cloud21.cvtt.vpn:5500/tester:latest
container_name: tester
environment:
- REDIS_HOST=redis
volumes:
- ./config:/config
- ./logs:/logs
- ./data:/data
depends_on:
- redis
- md_gateway
redis:
image: redis:latest
container_name: redis
ports:
- "6379:6379"
volumes:
- ./data/redis:/data
# test_program:
# image: your_test_program_image:latest
# container_name: test_program
# environment:
# - REDIS_HOST=redis
# volumes:
# - ./config:/shared/config
# - ./logs:/shared/logs
# depends_on:
# - market_data_gateway
# - redis
# volumes:
# redis_data:

View File

@ -0,0 +1,3 @@
.git
**/__pycache__
.pipenv

View File

@ -0,0 +1,25 @@
FROM python:3.12-slim
ARG ROOT_MCRSVC_DIR=docker_dev/microservices
ARG FROM_DIR=${ROOT_MCRSVC_DIR}/md_gateway
COPY ${ROOT_MCRSVC_DIR}/requirements.txt /
RUN pip install --upgrade pip --root-user-action=ignore
RUN pip install -r /requirements.txt --root-user-action=ignore
COPY cvttpy /cvttpy
# COPY ${ROOT_MCRSVC_DIR}/.creds /.creds
COPY ${FROM_DIR}/run_mdgw.sh /run_mdgw.sh
RUN chmod +x /run_mdgw.sh
RUN mkdir /logs
RUN mkdir -p /config
WORKDIR /
SHELL ["/bin/bash", "-c"]
ENV PYTHONPATH=/
ENTRYPOINT [ "/run_mdgw.sh" ]

View File

@ -0,0 +1,48 @@
#!/bin/bash
# runs in container
# runs on host to start container
usage() {
echo -n "Usage: ${0}"
echo -n " [-c <config (/config/cvtt.cfg)>]"
echo -n " [-a <admin_port (7220)>]"
echo -n " [-z (compress log)"]
echo
exit 1
}
Config=/config/cvtt.cfg
AdminPort=7220
COMPRESS_LOG=false
while getopts "c:a:z" opt; do
case ${opt} in
c ) Config=$OPTARG ;;
a ) AdminPort=$OPTARG ;;
z ) COMPRESS_LOG=true ;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
Cmd="python3.12"
Cmd="${Cmd} cvttpy/apps/microservices/market_gateways/md_gateway.py"
Cmd="${Cmd} --config=${Config}"
Cmd="${Cmd} --credentials_file=/.creds"
Cmd="${Cmd} --admin_port=${AdminPort}"
Cmd="${Cmd} --log_file=/logs/%T.md_gateway.log"
if ${COMPRESS_LOG} ; then
Cmd="${Cmd} --compress_log"
fi
echo ${Cmd}
eval ${Cmd}

View File

@ -0,0 +1,3 @@
.git
**/__pycache__
.pipenv

View File

@ -0,0 +1,25 @@
FROM python:3.12-slim
ARG ROOT_MCRSVC_DIR=docker_dev/microservices
ARG FROM_DIR=${ROOT_MCRSVC_DIR}/oe_gateway
COPY ${ROOT_MCRSVC_DIR}/requirements.txt /
RUN pip install --upgrade pip --root-user-action=ignore
RUN pip install -r /requirements.txt --root-user-action=ignore
COPY cvttpy /cvttpy
COPY ${ROOT_MCRSVC_DIR}/.creds /.creds
COPY ${FROM_DIR}/run_oegw.sh /run_oegw.sh
RUN chmod +x /run_oegw.sh
RUN mkdir /logs
RUN mkdir -p /config
WORKDIR /
SHELL ["/bin/bash", "-c"]
ENV PYTHONPATH=/
ENTRYPOINT [ "/run_oegw.sh" ]

View File

@ -0,0 +1,48 @@
#!/bin/bash
# runs in container
# runs on host to start container
usage() {
echo -n "Usage: ${0}"
echo -n " [-c <config (/config/cvtt.cfg)>]"
echo -n " [-a <admin_port (7220)>]"
echo -n " [-z (compress log)"]
echo
exit 1
}
Config=/config/cvtt.cfg
AdminPort=7220
COMPRESS_LOG=false
while getopts "c:a:z" opt; do
case ${opt} in
c ) Config=$OPTARG ;;
a ) AdminPort=$OPTARG ;;
z ) COMPRESS_LOG=true ;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
Cmd="python3.12"
Cmd="${Cmd} cvttpy/apps/microservices/market_gateways/order_entry_gateway.py"
Cmd="${Cmd} --config=${Config}"
Cmd="${Cmd} --credentials_file=/.creds"
Cmd="${Cmd} --admin_port=${AdminPort}"
Cmd="${Cmd} --log_file=/logs/%T.md_gateway.log"
if ${COMPRESS_LOG} ; then
Cmd="${Cmd} --compress_log"
fi
echo ${Cmd}
eval ${Cmd}

View File

@ -0,0 +1,7 @@
aiohttp>=3.7.4.post0
nest-asyncio>=1.5.5
hjson>=3.1.0
sortedcontainers>=2.4.0
redis>=5.0.8
python-dateutil>=2.8.2
types-python-dateutil>=2.8.19.6

View File

@ -0,0 +1,3 @@
.git
**/__pycache__
.pipenv

View File

@ -0,0 +1,26 @@
FROM python:3.12-slim
ARG ROOT_MUSVC_DIR=docker_dev/microservices
ARG FROM_DIR=${ROOT_MUSVC_DIR}/tester
COPY ${ROOT_MUSVC_DIR}/requirements.txt /
RUN pip install --upgrade pip --root-user-action=ignore
RUN pip install -r /requirements.txt --root-user-action=ignore
COPY cvttpy /cvttpy
# COPY ${ROOT_MUSVC_DIR}/.creds /.creds
COPY ${FROM_DIR}/run_tester.sh /run_tester.sh
RUN chmod +x /run_tester.sh
RUN mkdir /logs
RUN mkdir -p /config
RUN mkdir -p /data
WORKDIR /
SHELL ["/bin/bash", "-c"]
ENV PYTHONPATH=/
ENTRYPOINT [ "/run_tester.sh" ]

View File

@ -0,0 +1,55 @@
#!/bin/bash
# runs in container
# runs on host to start container
usage() {
echo -n "Usage: ${0}"
echo -n " [-c <config (/config/cvtt.cfg) >]"
echo -n " [-i <instruments (COINBASE:PAIR-BTC-USD,BNBFUT:PERP-BTC-USDT) >]"
echo -n " [-d <data types (TRADES) >]"
echo -n " [-a <admin_port (7221) >]"
echo -n " [-z (compress log (false))"]
echo
exit 1
}
Config=/config/cvtt.cfg
AdminPort=7221
COMPRESS_LOG=false
Instruments="COINBASE:PAIR-BTC-USD,BNBFUT:PERP-BTC-USDT"
DataTypes="TRADES"
while getopts "a:c:d:i:z" opt; do
case ${opt} in
c ) Config=$OPTARG ;;
a ) AdminPort=$OPTARG ;;
i ) Instruments=$OPTARG ;;
d ) DataTypes=$OPTARG ;;
z ) COMPRESS_LOG=true ;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
Cmd="python3.12"
Cmd="${Cmd} cvttpy/apps/microservices/tester/musvc_tester.py"
Cmd="${Cmd} --config=${Config}"
Cmd="${Cmd} --credentials_file=/.creds"
Cmd="${Cmd} --admin_port=${AdminPort}"
Cmd="${Cmd} --data_types=${DataTypes}"
Cmd="${Cmd} --instruments=${Instruments}"
Cmd="${Cmd} --log_file=/logs/%T.tester.log"
if ${COMPRESS_LOG} ; then
Cmd="${Cmd} --compress_log"
fi
echo ${Cmd}
eval ${Cmd}

View File

@ -0,0 +1,21 @@
FROM python:3.12-slim
ARG FROM_DIR=docker_dev/test/rsync_test
COPY ${FROM_DIR}/test.sh /test.sh
RUN apt-get update && apt-get install -y rsync openssh-client
COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub
# Shared Volumes
RUN chmod +x /test.sh
WORKDIR /
SHELL ["/bin/bash", "-c"]
ENV PYTHONPATH=/
ENTRYPOINT [ "/test.sh" ]

19
test/rsync_test/test.sh Executable file
View File

@ -0,0 +1,19 @@
#!/bin/bash
date > /testfile.txt
Source=/testfile.txt
Targets=
Targets="${Targets} cvtt@cloud21.cvtt.vpn:/tmp/"
Targets="${Targets} cvtt@hs01.cvtt.vpn:/tmp/"
for tgt in ${Targets}
do
Cmd="/usr/bin/rsync -ahv"
Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
Cmd+=" ${Source} ${tgt}"
echo $Cmd
eval $Cmd
done