rearrange + md monitor
This commit is contained in:
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"__dummy__": null
|
||||
, "ALPACA_SANDBOX": {
|
||||
"api_key": "PKLZSLFZMFMN1R28K9HK"
|
||||
, "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
FROM python:3.10-slim
|
||||
|
||||
ARG FROM_DIR=docker_dev/market_data/alpaca_md_day
|
||||
|
||||
# Update the package list and install required packages
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
apt-utils \
|
||||
software-properties-common \
|
||||
curl \
|
||||
rsync \
|
||||
openssh-client \
|
||||
parallel \
|
||||
jq
|
||||
|
||||
RUN pip install --upgrade pip
|
||||
|
||||
|
||||
COPY ${FROM_DIR}/requirements.txt /
|
||||
RUN pip install -r /requirements.txt
|
||||
|
||||
COPY cvttpy /cvttpy
|
||||
COPY ${FROM_DIR}/.creds /.creds
|
||||
COPY ${FROM_DIR}/.creds /root/.creds
|
||||
COPY ${FROM_DIR}/alpaca_md_day.sh /alpaca_md_day.sh
|
||||
|
||||
COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
|
||||
COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
|
||||
COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
|
||||
|
||||
RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
|
||||
|
||||
|
||||
# Shared Volumes
|
||||
RUN mkdir -p /app/data
|
||||
RUN mkdir /logs
|
||||
RUN chmod +x /alpaca_md_day.sh
|
||||
|
||||
WORKDIR /
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
ENV PYTHONPATH=/
|
||||
ENTRYPOINT [ "/alpaca_md_day.sh" ]
|
||||
|
||||
# CMD [ "echo", "alpaca_md_day"]
|
||||
@@ -0,0 +1,116 @@
|
||||
# Build/Dev Host
|
||||
## Build
|
||||
```bash
|
||||
Version=$(cat /home/oleg/develop/cvtt2/cvttpy/release_version.txt)
|
||||
|
||||
cd /home/oleg/develop/cvtt2
|
||||
docker build -t crypto_md_day -t crypto_md_day:${Version} -f docker_dev/crypto_md_day/Dockerfile .
|
||||
|
||||
## Deploy
|
||||
|
||||
/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day
|
||||
|
||||
```
|
||||
|
||||
## On target Machine make sure user can use docker
|
||||
```bash
|
||||
sudo usermod -a -G docker cvtt
|
||||
```
|
||||
**Re-login as cvtt after adding cvtt to the group**
|
||||
|
||||
### Clean
|
||||
```bash
|
||||
docker rm -f crypto_md_day
|
||||
docker images --filter=reference=crypto_md_day --format="{{.ID}}" | uniq | xargs docker rmi -f
|
||||
```
|
||||
### Test run locally
|
||||
```
|
||||
docker load -i /home/oleg/docker_images/crypto_md_day.img.tar
|
||||
mkdir -p /tmp/data
|
||||
mkdir -p /tmp/logs
|
||||
docker run --network="host" --name=crypto_md_day -v /tmp/data:/app/data -v /tmp/logs:/logs crypto_md_day
|
||||
```
|
||||
|
||||
## Restarting
|
||||
```bash
|
||||
docker restart crypto_md_day
|
||||
```
|
||||
|
||||
# Useful Commands
|
||||
```bash
|
||||
docker images
|
||||
docker ps -a
|
||||
docker image rm <image>
|
||||
docker rm <container>
|
||||
docker exec -it <container> /bin/bash
|
||||
```
|
||||
|
||||
---
|
||||
```bash
|
||||
docker rm -f crypto_md_day
|
||||
docker images --filter=reference=crypto_md_day --format="{{.ID}}" | uniq | xargs docker rmi -f
|
||||
|
||||
/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day cryptoval4.cvtt.vpn
|
||||
docker tag crypto_md_day:latest cloud21.cvtt.vpn:5500/crypto_md_day:latest
|
||||
docker push cloud21.cvtt.vpn:5500/crypto_md_day:latest
|
||||
|
||||
# Local test
|
||||
docker load -i /home/oleg/docker_images/crypto_md_day.img.tar && docker run --network="host" --name=crypto_md_day -v /tmp/data:/app/data crypto_md_day -h cvttdata -d 20240717 -s coinbase
|
||||
```
|
||||
|
||||
## Local (cloud21) docker registry
|
||||
### Run registry container (cloud21)
|
||||
|
||||
|
||||
```bash
|
||||
mkdir /opt/store/cvtt/docker_registry/data
|
||||
vi /opt/store/cvtt/docker_registry/config.yml
|
||||
```
|
||||
|
||||
```yaml
|
||||
version: 0.1
|
||||
log:
|
||||
fields:
|
||||
service: registry
|
||||
storage:
|
||||
filesystem:
|
||||
rootdirectory: /var/lib/registry
|
||||
http:
|
||||
addr: :5500
|
||||
|
||||
```
|
||||
```bash
|
||||
docker run -d -p 5500:5500 --name registry -v /opt/store/cvtt/docker_registry/config.yml:/etc/docker/registry/config.yml -v /opt/store/cvtt/docker_registry/data:/var/lib/registry registry:2
|
||||
```
|
||||
|
||||
### Building
|
||||
#### /etc/docker/daemon.json
|
||||
```json
|
||||
{
|
||||
"insecure-registries": ["cloud21.cvtt.vpn:5500"]
|
||||
}
|
||||
```
|
||||
```bash
|
||||
sudo systemctl restart docker
|
||||
```
|
||||
```bash
|
||||
/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day cryptoval4.cvtt.vpn
|
||||
docker tag crypto_md_day:latest cloud21.cvtt.vpn:5500/crypto_md_day:latest
|
||||
docker push cloud21.cvtt.vpn:5500/crypto_md_day:latest
|
||||
```
|
||||
|
||||
### Loading and Running (cryptoval4)
|
||||
#### /etc/docker/daemon.json
|
||||
```json
|
||||
{
|
||||
"insecure-registries": ["cloud21.cvtt.vpn:5500"]
|
||||
}
|
||||
```
|
||||
```bash
|
||||
sudo systemctl restart docker
|
||||
```
|
||||
```
|
||||
docker pull cloud21.cvtt.vpn:5500/crypto_md_day:latest
|
||||
docker run --network="host" --name=cloud21.cvtt.vpn:5500/crypto_md_day crypto_md_day -h cvttdata -d 20240717 -s coinbase
|
||||
```
|
||||
|
||||
Executable
+138
@@ -0,0 +1,138 @@
|
||||
#!/bin/bash
|
||||
|
||||
# --- Settings
|
||||
export PYTHONPATH=/
|
||||
export Python=python3.10
|
||||
export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
|
||||
export PyScript=/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
|
||||
|
||||
export OutputDir=/app/data/alpaca_md # Local
|
||||
export LogDir=/logs
|
||||
|
||||
DEFAULT_NUM_JOBS=10
|
||||
SLEEP_SEC_AFTER_PROC=2
|
||||
DEFAULT_SLICE_SIZE=500
|
||||
# --- Settings
|
||||
|
||||
mkdir -p ${OutputDir}
|
||||
mkdir -p ${LogDir}
|
||||
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <date (YYYY-MM-DD)> [<num-jobs> (30) ] [<instrument_list_file>] "
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo "CommandLine: ${*}"
|
||||
|
||||
Start=${1}
|
||||
NumJobs=${2}
|
||||
|
||||
export CalendarURL=http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
|
||||
|
||||
is_business_day() {
|
||||
dt=${1}
|
||||
|
||||
open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
|
||||
if [ -n "${open_time}" ]; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
|
||||
}
|
||||
export -f is_business_day
|
||||
|
||||
|
||||
if [ -z ${Start} ] ; then
|
||||
echo "start is not specified, getting yesterday..."
|
||||
Start=$(date -d "yesterday" "+%Y-%m-%d")
|
||||
echo "Start=${Start}"
|
||||
fi
|
||||
|
||||
while true; do
|
||||
if is_business_day ${Start}; then
|
||||
break
|
||||
fi
|
||||
echo "${Start} is not business day in US"
|
||||
Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
|
||||
done
|
||||
|
||||
End=${Start} # only 1 day is practical
|
||||
|
||||
if [ "${NumJobs}" == "" ] ; then
|
||||
NumJobs=${DEFAULT_NUM_JOBS}
|
||||
fi
|
||||
|
||||
echo "Start=${Start} End=${End} NumJobs=${NumJobs}"
|
||||
|
||||
|
||||
run_proc() {
|
||||
Inst=${1}
|
||||
Start=${2}
|
||||
End=${3}
|
||||
echo "Running for $Inst"
|
||||
|
||||
Cmd="${Python} ${PyScript}"
|
||||
Cmd="${Cmd} --config=${Config}"
|
||||
Cmd="${Cmd} --output_dir=${OutputDir}"
|
||||
Cmd="${Cmd} --instruments=ALPACA:${Inst}"
|
||||
Cmd="${Cmd} --start=${Start}"
|
||||
if [ "${End}" != "" ]; then
|
||||
Cmd="${Cmd} --end=${End}"
|
||||
fi
|
||||
Cmd="${Cmd} --log_file=${LogDir}/${Inst}.log"
|
||||
Cmd="${Cmd} --log_level=WARNING"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
sleep 2 #${SLEEP_SEC_AFTER_PROC}
|
||||
}
|
||||
|
||||
export -f run_proc
|
||||
|
||||
key=$(jq -r '.["ALPACA_SANDBOX"] | .api_key' ~/.creds)
|
||||
secret=$(jq -r '.["ALPACA_SANDBOX"] | .secret_key' ~/.creds)
|
||||
|
||||
Cmd="curl -s --request GET --url 'https://paper-api.alpaca.markets/v2/assets?status=active'"
|
||||
Cmd="${Cmd} --header 'APCA-API-KEY-ID: ${key}'"
|
||||
Cmd="${Cmd} --header 'APCA-API-SECRET-KEY: ${secret}'"
|
||||
Cmd="${Cmd} --header 'accept: application/json'"
|
||||
Cmd="${Cmd} | jq '.[] | select(.class == \"us_equity\" and .exchange != \"OTC\") | .symbol'"
|
||||
Cmd="${Cmd} | sed 's/\"//g'"
|
||||
Cmd="${Cmd} | sed 's/^/STOCK-/'"
|
||||
# split string into array
|
||||
Instruments=()
|
||||
for Inst in $(eval ${Cmd})
|
||||
do
|
||||
Instruments+=("$Inst")
|
||||
done
|
||||
|
||||
slice_size=${DEFAULT_SLICE_SIZE} # 10K symbols parallel cannot handle
|
||||
for ((ii=0; ii <${#Instruments[@]}; ii+=slice_size)); do
|
||||
InstSlice=("${Instruments[@]:ii:slice_size}")
|
||||
|
||||
parallel -j ${NumJobs} run_proc {} ${Start} ${End} ::: "${InstSlice[@]}"
|
||||
done
|
||||
|
||||
echo "Compressing"
|
||||
for file in $(find ${OutputDir} -type f -name '*db' -print )
|
||||
do
|
||||
echo "Compressing ${file}"
|
||||
gzip ${file}
|
||||
done
|
||||
|
||||
|
||||
Source=/app/data/
|
||||
Targets=
|
||||
Targets="${Targets} oleg@homestore.cvtt.vpn:/works/cvtt/md_archive/equity"
|
||||
Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
|
||||
|
||||
|
||||
for tgt in ${Targets}
|
||||
do
|
||||
Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
|
||||
echo $Cmd
|
||||
eval $Cmd
|
||||
done
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
aiohttp>=3.7.4.post0
|
||||
nest-asyncio>=1.5.5
|
||||
asyncpg>=0.27.0
|
||||
hjson>=3.1.0
|
||||
pandas>=1.5.3
|
||||
sortedcontainers>=2.4.0
|
||||
aioredis>=2.0.1
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"__dummy__": null
|
||||
, "TSDB_MD_CLD21_RO": {
|
||||
"host": "cloud21.cvtt.vpn"
|
||||
, "port": 5432
|
||||
, "user": "cvtt_ro"
|
||||
, "database": "cvtt_md"
|
||||
, "password": "3Gkotqa0ZSmZLXSH3RkTUm"
|
||||
}
|
||||
, "TSDB_MD_CVTTDATA_RO": {
|
||||
"host": "cvttdata.cvtt.vpn"
|
||||
, "port": 5432
|
||||
, "user": "cvtt_ro"
|
||||
, "database": "cvtt_md"
|
||||
, "password": "3Gkotqa0ZSmZLXSH3RkTUm"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
FROM python:3.10-slim
|
||||
|
||||
ARG FROM_DIR=docker_dev/market_data/crypto_md_day
|
||||
|
||||
COPY ${FROM_DIR}/requirements.txt /
|
||||
RUN pip install --upgrade pip --root-user-action=ignore
|
||||
RUN pip install -r /requirements.txt --root-user-action=ignore
|
||||
|
||||
COPY cvttpy /cvttpy
|
||||
COPY ${FROM_DIR}/.creds /.creds
|
||||
COPY ${FROM_DIR}/crypto_md_day.sh /crypto_md_day.sh
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends rsync openssh-client && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
|
||||
COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
|
||||
COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
|
||||
|
||||
RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
|
||||
|
||||
|
||||
# Shared Volumes
|
||||
RUN mkdir -p /app/data
|
||||
RUN mkdir /logs
|
||||
RUN chmod +x /crypto_md_day.sh
|
||||
|
||||
WORKDIR /
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
ENV PYTHONPATH=/
|
||||
ENTRYPOINT [ "/crypto_md_day.sh" ]
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
# Build/Dev Host
|
||||
## Build
|
||||
```bash
|
||||
Version=$(cat /home/oleg/develop/cvtt2/cvttpy/release_version.txt)
|
||||
|
||||
cd /home/oleg/develop/cvtt2
|
||||
docker build -t crypto_md_day -t crypto_md_day:${Version} -f docker_dev/crypto_md_day/Dockerfile .
|
||||
|
||||
## Deploy
|
||||
|
||||
/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day
|
||||
|
||||
```
|
||||
|
||||
## On target Machine make sure user can use docker
|
||||
```bash
|
||||
sudo usermod -a -G docker cvtt
|
||||
```
|
||||
**Re-login as cvtt after adding cvtt to the group**
|
||||
|
||||
### Clean
|
||||
```bash
|
||||
docker rm -f crypto_md_day
|
||||
docker images --filter=reference=crypto_md_day --format="{{.ID}}" | uniq | xargs docker rmi -f
|
||||
```
|
||||
### Test run locally
|
||||
```
|
||||
docker load -i /home/oleg/docker_images/crypto_md_day.img.tar
|
||||
mkdir -p /tmp/data
|
||||
mkdir -p /tmp/logs
|
||||
docker run --network="host" --name=crypto_md_day -v /tmp/data:/app/data -v /tmp/logs:/logs crypto_md_day
|
||||
```
|
||||
|
||||
## Restarting
|
||||
```bash
|
||||
docker restart crypto_md_day
|
||||
```
|
||||
|
||||
# Useful Commands
|
||||
```bash
|
||||
docker images
|
||||
docker ps -a
|
||||
docker image rm <image>
|
||||
docker rm <container>
|
||||
docker exec -it <container> /bin/bash
|
||||
```
|
||||
|
||||
---
|
||||
```bash
|
||||
docker rm -f crypto_md_day
|
||||
docker images --filter=reference=crypto_md_day --format="{{.ID}}" | uniq | xargs docker rmi -f
|
||||
|
||||
/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day cryptoval4.cvtt.vpn
|
||||
docker tag crypto_md_day:latest cloud21.cvtt.vpn:5500/crypto_md_day:latest
|
||||
docker push cloud21.cvtt.vpn:5500/crypto_md_day:latest
|
||||
|
||||
# Local test
|
||||
docker load -i /home/oleg/docker_images/crypto_md_day.img.tar && docker run --network="host" --name=crypto_md_day -v /tmp/data:/app/data crypto_md_day -h cvttdata -d 20240717 -s coinbase
|
||||
```
|
||||
|
||||
## Local (cloud21) docker registry
|
||||
### Run registry container (cloud21)
|
||||
|
||||
|
||||
```bash
|
||||
mkdir /opt/store/cvtt/docker_registry/data
|
||||
vi /opt/store/cvtt/docker_registry/config.yml
|
||||
```
|
||||
|
||||
```yaml
|
||||
version: 0.1
|
||||
log:
|
||||
fields:
|
||||
service: registry
|
||||
storage:
|
||||
filesystem:
|
||||
rootdirectory: /var/lib/registry
|
||||
http:
|
||||
addr: :5500
|
||||
|
||||
```
|
||||
```bash
|
||||
docker run -d -p 5500:5500 --name registry -v /opt/store/cvtt/docker_registry/config.yml:/etc/docker/registry/config.yml -v /opt/store/cvtt/docker_registry/data:/var/lib/registry registry:2
|
||||
```
|
||||
|
||||
### Building
|
||||
#### /etc/docker/daemon.json
|
||||
```json
|
||||
{
|
||||
"insecure-registries": ["cloud21.cvtt.vpn:5500"]
|
||||
}
|
||||
```
|
||||
```bash
|
||||
sudo systemctl restart docker
|
||||
```
|
||||
```bash
|
||||
/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day cryptoval4.cvtt.vpn
|
||||
docker tag crypto_md_day:latest cloud21.cvtt.vpn:5500/crypto_md_day:latest
|
||||
docker push cloud21.cvtt.vpn:5500/crypto_md_day:latest
|
||||
```
|
||||
|
||||
### Loading and Running (cryptoval4)
|
||||
#### /etc/docker/daemon.json
|
||||
```json
|
||||
{
|
||||
"insecure-registries": ["cloud21.cvtt.vpn:5500"]
|
||||
}
|
||||
```
|
||||
```bash
|
||||
sudo systemctl restart docker
|
||||
```
|
||||
```
|
||||
docker pull cloud21.cvtt.vpn:5500/crypto_md_day:latest
|
||||
docker run --network="host" --name=cloud21.cvtt.vpn:5500/crypto_md_day crypto_md_day -h cvttdata -d 20240717 -s coinbase
|
||||
```
|
||||
|
||||
Executable
+81
@@ -0,0 +1,81 @@
|
||||
#!/bin/bash
|
||||
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 -h <db host> -d <YYYYMMDD Date> -s <schemas|coinbase,bnbspot,bnbfut>"
|
||||
exit 1
|
||||
}
|
||||
|
||||
DbHost=""
|
||||
Date=""
|
||||
Schemas=coinbase,bnbspot,bnbfut
|
||||
|
||||
while getopts ":h:d:s:" opt; do
|
||||
case ${opt} in
|
||||
h )
|
||||
DbHost=$OPTARG
|
||||
;;
|
||||
d )
|
||||
Date=$OPTARG
|
||||
;;
|
||||
s )
|
||||
Schemas=$OPTARG
|
||||
;;
|
||||
\? )
|
||||
echo "Invalid option: -$OPTARG" >&2
|
||||
usage
|
||||
;;
|
||||
: )
|
||||
echo "Option -$OPTARG requires an argument." >&2
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "${DbHost}" ] || [ -z "$Date" ]; then
|
||||
echo "DbHost=${DbHost} Date=${Date}. Cmdline: ${*}"
|
||||
usage
|
||||
fi
|
||||
|
||||
CredKey=""
|
||||
if [ "${DbHost}" == "cvttdata" ] ; then
|
||||
CredKey=TSDB_MD_CVTTDATA_RO
|
||||
elif [ "${DbHost}" == "cloud21" ] ; then
|
||||
CredKey=TSDB_MD_CLD21_RO
|
||||
else
|
||||
echo "host ${DbHost} is not valid source"
|
||||
usage
|
||||
fi
|
||||
|
||||
export PYTHONPATH=/
|
||||
|
||||
Cmd="python3.10"
|
||||
Cmd="${Cmd} cvttpy/research/utils/archive_ts_md.py"
|
||||
Cmd="${Cmd} --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder"
|
||||
Cmd="${Cmd} --db_credentials_key=${CredKey}"
|
||||
Cmd="${Cmd} --credentials_file=/.creds"
|
||||
Cmd="${Cmd} --date=${Date}"
|
||||
Cmd="${Cmd} --schemas=${Schemas}"
|
||||
Cmd="${Cmd} --root_dir=/app/data/${DbHost}"
|
||||
Cmd="${Cmd} --format=SQLite"
|
||||
Cmd="${Cmd} --compress"
|
||||
# Cmd="${Cmd} --log_file=/logs/%T.crypto_md_day.log"
|
||||
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
echo "Data Sync"
|
||||
|
||||
Source=/app/data/
|
||||
Targets=
|
||||
Targets="${Targets} oleg@homestore.cvtt.vpn:/works/cvtt/md_archive/crypto"
|
||||
Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto"
|
||||
|
||||
|
||||
for tgt in ${Targets}
|
||||
do
|
||||
Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
|
||||
echo $Cmd
|
||||
eval $Cmd
|
||||
done
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
aiohttp>=3.7.4.post0
|
||||
nest-asyncio>=1.5.5
|
||||
asyncpg>=0.27.0
|
||||
hjson>=3.1.0
|
||||
pandas>=1.5.3
|
||||
sortedcontainers>=2.4.0
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"__dummy__": null
|
||||
, "TSDB_MD_CLD21": {
|
||||
"host": "cloud21.cvtt.vpn"
|
||||
, "port": 5432
|
||||
, "user": "cvtt"
|
||||
, "database": "cvtt_md"
|
||||
, "password": "ICdIh0JnMM7vM7Pf"
|
||||
}
|
||||
, "TSDB_MD_CVTTDATA": {
|
||||
"host": "cvttdata.cvtt.vpn"
|
||||
, "port": 5432
|
||||
, "user": "cvtt"
|
||||
, "database": "cvtt_md"
|
||||
, "password": "ICdIh0JnMM7vM7Pf"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
FROM python:3.10-slim
|
||||
|
||||
ARG FROM_DIR=docker_dev/market_data/md_recorder
|
||||
|
||||
|
||||
COPY ${FROM_DIR}/requirements.txt /
|
||||
RUN pip install --upgrade pip --root-user-action=ignore
|
||||
RUN pip install -r /requirements.txt --root-user-action=ignore
|
||||
|
||||
COPY cvttpy /cvttpy
|
||||
COPY ${FROM_DIR}/.creds /.creds
|
||||
COPY ${FROM_DIR}/run_md_recorder.sh /run_md_recorder.sh
|
||||
|
||||
|
||||
# Shared Volumes
|
||||
RUN mkdir -p /app/data
|
||||
RUN mkdir /logs
|
||||
RUN chmod +x /run_md_recorder.sh
|
||||
|
||||
WORKDIR /
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
ENV PYTHONPATH=/
|
||||
ENTRYPOINT [ "/run_md_recorder.sh" ]
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
aiohttp>=3.7.4.post0
|
||||
nest-asyncio>=1.5.5
|
||||
asyncpg>=0.27.0
|
||||
hjson>=3.1.0
|
||||
pandas>=1.5.3
|
||||
sortedcontainers>=2.4.0
|
||||
aioredis>=2.0.1
|
||||
Executable
+84
@@ -0,0 +1,84 @@
|
||||
#!/bin/bash
|
||||
|
||||
# runs in container
|
||||
|
||||
ValidJobs=('BNBFUT_CLD21' 'BNBSPOT_CLD21' 'COINBASE_CLD21' 'BNBFUT_CVTTDATA' 'BNBSPOT_CVTTDATA' 'COINBASE_CVTTDATA')
|
||||
# runs on host to start container
|
||||
usage() {
|
||||
echo "Usage: $0 <job_name; one of (${ValidJobs[@]})>"
|
||||
exit 1
|
||||
}
|
||||
|
||||
is_valid() {
|
||||
local job=$1
|
||||
for valid_job in "${ValidJobs[@]}";
|
||||
do
|
||||
# echo "job=$job valid_job=$valid_job"
|
||||
if [[ "${job}" == "${valid_job}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
job=${1}
|
||||
if ! is_valid "${job}"; then
|
||||
usage
|
||||
fi
|
||||
|
||||
# ConfigServer=cloud16.cvtt.vpn
|
||||
ConfigServer=cloud23.cvtt.vpn:6789
|
||||
|
||||
Cmd="python3.10"
|
||||
Cmd="${Cmd} cvttpy/apps/md/md_recorder.py"
|
||||
Cmd="${Cmd} --config=http://${ConfigServer}/apps/md_recorder"
|
||||
if [ "${job}" == "BNBFUT_CLD21" ] ; then
|
||||
Cmd="${Cmd} --active_exchanges=BNBFUT"
|
||||
Cmd="${Cmd} --instruments=BNBFUT:PERP-BTC-USDT,BNBFUT:PERP-ETH-USDT"
|
||||
Cmd="${Cmd} --credentials_file=/.creds"
|
||||
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
|
||||
Cmd="${Cmd} --admin_port=7201"
|
||||
Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBFUT.CLD21.log"
|
||||
elif [ "${job}" == "COINBASE_CLD21" ] ; then
|
||||
Cmd="${Cmd} --active_exchanges=COINBASE_AT"
|
||||
Cmd="${Cmd} --instruments=COINBASE:PAIR-BTC-USD,COINBASE:PAIR-ETH-USD,COINBASE:PAIR-LTC-USD,COINBASE:PAIR-XRP-USD"
|
||||
Cmd="${Cmd} --credentials_file=/.creds"
|
||||
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
|
||||
Cmd="${Cmd} --admin_port=7202"
|
||||
Cmd="${Cmd} --log_file=/logs/%T.MD_REC.COINBASE.CLD21.log"
|
||||
elif [ "${job}" == "BNBSPOT_CLD21" ] ; then
|
||||
Cmd="${Cmd} --active_exchanges=BNBSPOT"
|
||||
Cmd="${Cmd} --instruments=BNBSPOT:PAIR-BTC-USDT,BNBSPOT:PAIR-ETH-USDT,BNBSPOT:PAIR-LTC-USDT,BNBSPOT:PAIR-XRP-USDT,BNBSPOT:PAIR-USDC-USDT"
|
||||
Cmd="${Cmd} --credentials_file=/.creds"
|
||||
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
|
||||
Cmd="${Cmd} --admin_port=7203"
|
||||
Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBSPOT.CLD21.log"
|
||||
elif [ "${job}" == "BNBSPOT_CVTTDATA" ] ; then
|
||||
Cmd="${Cmd} --active_exchanges=BNBSPOT"
|
||||
Cmd="${Cmd} --instruments=BNBSPOT:PAIR-BTC-USDT,BNBSPOT:PAIR-ETH-USDT,BNBSPOT:PAIR-LTC-USDT,BNBSPOT:PAIR-XRP-USDT,BNBSPOT:PAIR-USDC-USDT"
|
||||
Cmd="${Cmd} --credentials_file=/.creds"
|
||||
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
|
||||
Cmd="${Cmd} --admin_port=7204"
|
||||
Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBSPOT.CVTTDATA.log"
|
||||
elif [ "${job}" == "BNBFUT_CVTTDATA" ] ; then
|
||||
Cmd="${Cmd} --active_exchanges=BNBFUT"
|
||||
Cmd="${Cmd} --instruments=BNBFUT:PERP-BTC-USDT,BNBFUT:PERP-ETH-USDT"
|
||||
Cmd="${Cmd} --credentials_file=/.creds"
|
||||
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
|
||||
Cmd="${Cmd} --admin_port=7205"
|
||||
Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBFUT.CVTTDATA.log"
|
||||
elif [ "${job}" == "COINBASE_CVTTDATA" ] ; then
|
||||
Cmd="${Cmd} --active_exchanges=COINBASE_AT"
|
||||
Cmd="${Cmd} --instruments=COINBASE:PAIR-BTC-USD,COINBASE:PAIR-ETH-USD,COINBASE:PAIR-LTC-USD,COINBASE:PAIR-XRP-USD"
|
||||
Cmd="${Cmd} --credentials_file=/.creds"
|
||||
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
|
||||
Cmd="${Cmd} --admin_port=7206"
|
||||
Cmd="${Cmd} --log_file=/logs/%T.MD_REC.COINBASE.CVTTDATA.log"
|
||||
else
|
||||
echo "Unrecognized JOB: ${job}"
|
||||
exit 1
|
||||
fi
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"__dummy__": null
|
||||
, "MATTERMOST": {
|
||||
"url": "https://mattermost.cryptovaltrading.com"
|
||||
, "team": "CVTT"
|
||||
, "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
|
||||
}
|
||||
, "TSDB_MD_CLD21": {
|
||||
"host": "cloud21.cvtt.vpn"
|
||||
, "port": 5432
|
||||
, "user": "cvtt"
|
||||
, "database": "cvtt_md"
|
||||
, "password": "ICdIh0JnMM7vM7Pf"
|
||||
}
|
||||
, "TSDB_MD_CVTTDATA": {
|
||||
"host": "cvttdata.cvtt.vpn"
|
||||
, "port": 5432
|
||||
, "user": "cvtt"
|
||||
, "database": "cvtt_md"
|
||||
, "password": "ICdIh0JnMM7vM7Pf"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
FROM python:3.10-slim
|
||||
|
||||
ARG FROM_DIR=docker_dev/market_data/md_recorder_monitor
|
||||
|
||||
|
||||
COPY ${FROM_DIR}/requirements.txt /
|
||||
RUN pip install --upgrade pip --root-user-action=ignore
|
||||
RUN pip install -r /requirements.txt --root-user-action=ignore
|
||||
|
||||
COPY cvttpy /cvttpy
|
||||
COPY ${FROM_DIR}/.creds /.creds
|
||||
COPY ${FROM_DIR}/run_mdr_monitor.sh /run_mdr_monitor.sh
|
||||
|
||||
|
||||
# Shared Volumes
|
||||
RUN mkdir -p /app/data
|
||||
RUN mkdir /logs
|
||||
RUN chmod +x /run_mdr_monitor.sh
|
||||
|
||||
WORKDIR /
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
ENV PYTHONPATH=/
|
||||
ENTRYPOINT [ "/run_mdr_monitor.sh" ]
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
aiohttp>=3.7.4.post0
|
||||
nest-asyncio>=1.5.5
|
||||
asyncpg>=0.27.0
|
||||
hjson>=3.1.0
|
||||
pandas>=1.5.3
|
||||
sortedcontainers>=2.4.0
|
||||
aioredis>=2.0.1
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
#!/bin/bash
|
||||
|
||||
# runs in container
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <instance (CLD21,CVTTDATA)> [<admin_port (def. 7225)>]"
|
||||
exit 1
|
||||
}
|
||||
|
||||
ValidInstances=('CLD21' 'CVTTDATA')
|
||||
is_valid() {
|
||||
local inst=$1
|
||||
for valid_inst in "${ValidInstances[@]}";
|
||||
do
|
||||
if [[ "$inst" == "$valid_inst" ]]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
|
||||
Instance=${1}
|
||||
if ! is_valid "${Instance}"; then
|
||||
usage
|
||||
fi
|
||||
|
||||
AdminPort=7225
|
||||
if [ "${2}" != "" ]; then
|
||||
AdminPort=${2}
|
||||
fi
|
||||
|
||||
|
||||
|
||||
# ConfigServer=cloud16.cvtt.vpn
|
||||
ConfigServer=cloud23.cvtt.vpn:6789
|
||||
|
||||
Cmd="python3.10"
|
||||
Cmd="${Cmd} cvttpy/apps/md/md_rec_monitor.py"
|
||||
Cmd="${Cmd} --config=http://c${ConfigServer}/apps/md_rec_monitor"
|
||||
Cmd="${Cmd} --db=${Instance}"
|
||||
Cmd="${Cmd} --admin_port=${AdminPort}"
|
||||
Cmd="${Cmd} --log_file=/logs/%%T.MD_REC.MONITOR.${Instance}.log
|
||||
|
||||
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user