archive/docker_dev
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docker compose based md_recorder

Browse Source
This commit is contained in:
Oleg Sheynin 2025-04-09 17:55:33 -04:00
parent 25073c2696
commit f09865d3a3
16 changed files with 819 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
VERSION
View File

@ -1 +1 @@
0.2.6,improved build.sh for microservices
0.2.7,new md_recorder build

22
market_data/md_recorder/Dockerfile
View File

@ -1,27 +1,25 @@
FROM python:3.12-slim
ARG FROM_DIR=docker_dev/market_data/md_recorder
# libpq needed for Timescale
RUN apt-get update && apt-get install -y \
apt-utils \
libpq-dev \
build-essential
COPY ${FROM_DIR}/requirements.txt /
COPY requirements.txt /
RUN pip install --upgrade pip --root-user-action=ignore
RUN pip install -r /requirements.txt --root-user-action=ignore
COPY cvttpy /cvttpy
COPY ${FROM_DIR}/run_md_recorder.sh /run_md_recorder.sh
RUN mkdir -p /logs /config
# Shared Volumes
RUN mkdir -p /app/data
RUN mkdir /logs
RUN chmod +x /run_md_recorder.sh
WORKDIR /
SHELL ["/bin/bash", "-c"]
ENV PYTHONPATH=/
ENTRYPOINT [ "/run_md_recorder.sh" ]
ENV PYTHONPATH=/
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]

69
market_data/md_recorder/build.sh Executable file
View File

@ -0,0 +1,69 @@
#!/bin/bash
# --- Settings
RegistryService=cloud21.cvtt.vpn:5500
ImageName=md_recorder
BashFunctionCode=$(realpath $(dirname ${0}))/../../mu_svc_functions.sh
# --- Settings
usage() {
echo "Usage: $0 -V <image_version> -L <library>:<library_version>"
exit 1
}
# ---------------- cmdline
while getopts "V:L:" opt; do
case ${opt} in
V )
ImageVersion=$OPTARG
;;
L )
Library=$OPTARG
;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
# ---------------- cmdline
if [ "${ImageVersion}" == "" ]
then
usage
fi
Project=
ProjectVersion=
if [ "${Library}" != "" ] ; then
IFS=':' read -ra parts <<< "${Library}"
Project=${parts[0]}
ProjectVersion=${parts[1]}
fi
DockerDir=$(realpath $(dirname ${0}))
cd ${DockerDir}
source ${BashFunctionCode}
if [ "${Project}" != "" ]; then
get_project_version ${Project} ${ProjectVersion}
fi
function cleanup {
cd ${DockerDir}
rm -rf cvttpy
}
trap cleanup EXIT
build_docker_image ${ImageName} ${RegistryService} ${ImageVersion} ${Project} ${ProjectVersion}
# echo "**** D E B U G E X I T" && exit
echo "***** ${0} D O N E"

0
market_data/md_recorder/.creds → market_data/md_recorder/deployment/cloud24/.creds
View File

52
market_data/md_recorder/deployment/cloud24/.env Normal file
View File

@ -0,0 +1,52 @@
# ----- cloud24 settings
CVTT_USER=1001:1001
IMAGE_VERSION=0.0.2
CONFIG_FILE=/config/md_recorder.cfg
CREDS_FILE=/config/.creds
# bnspot-cvttdata
BSCD_CONTAINER_NAME=md_recorder.BNBSPOT_CVTTDATA
BSCD_INSTRUMENT_GROUP=BNBSPOT_CVTTDATA
BSCD_EXCHANGE_ACCT_NAME=BNBSPOT
BSCD_DB_CRED_KEY=TSDB_MD_CVTTDATA
BSCD_LOG_FILE=/logs/%T.MD_REC.${BSCD_INSTRUMENT_GROUP}.log
BSCD_ADD_ARGS=--compress_log
BSCD_ADMIN_PORT=7201
# bnspot-cloud21
BSCLD21_CONTAINER_NAME=md_recorder.BNBSPOT_CLD21
BSCLD21_INSTRUMENT_GROUP=BNBSPOT_CLD21
BSCLD21_EXCHANGE_ACCT_NAME=BNBSPOT
BSCLD21_DB_CRED_KEY=TSDB_MD_CLD21
BSCLD21_LOG_FILE=/logs/%T.MD_REC.${BSCLD21_INSTRUMENT_GROUP}.log
BSCLD21_ADD_ARGS=--compress_log
BSCLD21_ADMIN_PORT=7202
# bnbfut-cloud21
BFCLD21_CONTAINER_NAME=md_recorder.BNBFUT_CLD21
BFCLD21_INSTRUMENT_GROUP=BNBFUT_CLD21
BFCLD21_EXCHANGE_ACCT_NAME=BNBFUT
BFCLD21_DB_CRED_KEY=TSDB_MD_CLD21
BFCLD21_LOG_FILE=/logs/%T.MD_REC.${BFCLD21_INSTRUMENT_GROUP}.log
BFCLD21_ADD_ARGS=--compress_log
BFCLD21_ADMIN_PORT=7203
# coinbase-cloud21
CBCLD21_CONTAINER_NAME=md_recorder.COINBASE_CLD21
CBCLD21_INSTRUMENT_GROUP=COINBASE_CLD21
CBCLD21_EXCHANGE_ACCT_NAME=COINBASE_AT
CBCLD21_DB_CRED_KEY=TSDB_MD_CLD21
CBCLD21_LOG_FILE=/logs/%T.MD_REC.${CBCLD21_INSTRUMENT_GROUP}.log
CBCLD21_ADD_ARGS=--compress_log
CBCLD21_ADMIN_PORT=7204

86
market_data/md_recorder/deployment/cloud24/docker-compose.yml Normal file
View File

@ -0,0 +1,86 @@
# CVTT Microservices
services:
bfcld21_md_recorder:
image: cloud21.cvtt.vpn:5500/md_recorder:${IMAGE_VERSION}
container_name: ${BFCLD21_CONTAINER_NAME}
user: ${CVTT_USER:-1001:1001}
pull_policy: always
restart: unless-stopped
environment:
CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg}
CREDS_FILE: ${CREDS_FILE}
EXCHANGE_ACCT_NAME: ${BFCLD21_EXCHANGE_ACCT_NAME}
INSTRUMENT_GROUP: ${BFCLD21_INSTRUMENT_GROUP}
DB_CRED_KEY: ${BFCLD21_DB_CRED_KEY}
LOG_FILE: ${BFCLD21_LOG_FILE}
ADD_ARGS: ${BFCLD21_ADD_ARGS:-}
volumes:
- ./config:/config
- ./logs:/logs
ports:
- ${BFCLD21_ADMIN_PORT}:7200
cbcld21_md_recorder:
image: cloud21.cvtt.vpn:5500/md_recorder:${IMAGE_VERSION}
container_name: ${CBCLD21_CONTAINER_NAME}
user: ${CVTT_USER:-1001:1001}
pull_policy: always
restart: unless-stopped
environment:
CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg}
CREDS_FILE: ${CREDS_FILE}
EXCHANGE_ACCT_NAME: ${CBCLD21_EXCHANGE_ACCT_NAME}
INSTRUMENT_GROUP: ${CBCLD21_INSTRUMENT_GROUP}
DB_CRED_KEY: ${CBCLD21_DB_CRED_KEY}
LOG_FILE: ${CBCLD21_LOG_FILE}
ADD_ARGS: ${CBCLD21_ADD_ARGS:-}
volumes:
- ./config:/config
- ./logs:/logs
ports:
- ${CBCLD21_ADMIN_PORT}:7200
bscld21_md_recorder:
image: cloud21.cvtt.vpn:5500/md_recorder:${IMAGE_VERSION}
container_name: ${BSCLD21_CONTAINER_NAME}
user: ${CVTT_USER:-1001:1001}
pull_policy: always
restart: unless-stopped
environment:
CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg}
CREDS_FILE: ${CREDS_FILE}
EXCHANGE_ACCT_NAME: ${BSCLD21_EXCHANGE_ACCT_NAME}
INSTRUMENT_GROUP: ${BSCLD21_INSTRUMENT_GROUP}
DB_CRED_KEY: ${BSCLD21_DB_CRED_KEY}
LOG_FILE: ${BSCLD21_LOG_FILE}
ADD_ARGS: ${BSCLD21_ADD_ARGS:-}
volumes:
- ./config:/config
- ./logs:/logs
ports:
- ${BSCLD21_ADMIN_PORT}:7200
bscd_md_recorder:
image: cloud21.cvtt.vpn:5500/md_recorder:${IMAGE_VERSION}
container_name: ${BSCD_CONTAINER_NAME}
user: ${CVTT_USER:-1001:1001}
pull_policy: always
restart: unless-stopped
environment:
CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg}
CREDS_FILE: ${CREDS_FILE}
EXCHANGE_ACCT_NAME: ${BSCD_EXCHANGE_ACCT_NAME}
INSTRUMENT_GROUP: ${BSCD_INSTRUMENT_GROUP}
DB_CRED_KEY: ${BSCD_DB_CRED_KEY}
LOG_FILE: ${BSCD_LOG_FILE}
ADD_ARGS: ${BSCD_ADD_ARGS:-}
volumes:
- ./config:/config
- ./logs:/logs
ports:
- ${BSCD_ADMIN_PORT}:7200

206
market_data/md_recorder/deployment/cloud24/md_recorder.cfg Normal file
View File

@ -0,0 +1,206 @@
{
"refdata": {
"assets": @inc=http://@env{CONFIG_SERVICE}/refdata/assets
, "instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/instruments
, "exchange_instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/exchange_instruments
, "dynamic_instrument_exchanges": ["ALPACA"]
, "exchanges": {
"COINBASE_AT": {
"exchange_id": "COINBASE"
, "name": "Coinbase Advanced Trade"
, "type": "SPOT"
, "mktdata" : {
# ****** NEW VAlUE
# "class": "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMultiWsFeed"
"class": "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMdFeed"
# ****** NEW VAlUE
, "reconnect_secs": 2
, "max_connections": 10
, "price_throttle": true
, "ws_url": ["WSS", "wss://ws-feed.exchange.coinbase.com"]
}
}
, "BNBSPOT": {
"exchange_id": "BNBSPOT"
, "name": "Binance Spot"
, "type": "SPOT"
, "mktdata" : {
# -------- NEW VALUE
# "class": "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMultiWsFeed"
"class": "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMktDataFeed"
# -------- NEW VALUE
, "max_connections": 10
, "reconnect_secs": 2.0
, "ws_url": ["WSS", "wss://stream.binance.com/ws"]
, "REST": {
"depth_snapshot_url": ["GET", "https://api.binance.com/api/v3/depth"],
}
}
}
, "BNBFUT": {
"exchange_id": "BNBFUT"
, "name": "Binance Futures USD-M"
, "type": "FUTURES"
, "mktdata" : {
# -------- NEW VALUE
# "class": "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMultiWsFeed"
"class": "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMktDataFeed"
# -------- NEW VALUE
, "max_connections": 10
, "reconnect_secs": 2.0
, "ws_url": ["WSS", "wss://fstream.binance.com/ws"]
, "REST": {
# https://fapi.binance.com/fapi/v1/depth?symbol=BNBBTC&limit=1000
"depth_snapshot_url": ["GET", "https://fapi.binance.com/fapi/v1/depth"],
}
}
}
}
}
, "comm": @inc=http://@env{CONFIG_SERVICE}/apps/common/comm
, "md_recorder": {
"instrument_groups": {
"BNBFUT_CLD21": [
"BNBFUT:PERP-BTC-USDT"
, "BNBFUT:PERP-ETH-USDT"
]
"BNBFUT_CVTTDATA": [
"BNBFUT:PERP-BTC-USDT"
, "BNBFUT:PERP-ETH-USDT"
]
, "BNBSPOT_CLD21": [
"BNBSPOT:PAIR-ADA-USDT"
, "BNBSPOT:PAIR-BCH-USDT"
, "BNBSPOT:PAIR-BTC-USDT"
, "BNBSPOT:PAIR-DOT-USDT"
, "BNBSPOT:PAIR-ETH-USDT"
, "BNBSPOT:PAIR-LTC-USDT"
, "BNBSPOT:PAIR-SOL-USDT"
, "BNBSPOT:PAIR-USDC-USDT"
, "BNBSPOT:PAIR-XRP-USDT"
]
, "BNBSPOT_CVTTDATA": [
"BNBSPOT:PAIR-ADA-USDT"
, "BNBSPOT:PAIR-BCH-USDT"
, "BNBSPOT:PAIR-BTC-USDT"
, "BNBSPOT:PAIR-DOT-USDT"
, "BNBSPOT:PAIR-ETH-USDT"
, "BNBSPOT:PAIR-LTC-USDT"
, "BNBSPOT:PAIR-SOL-USDT"
, "BNBSPOT:PAIR-USDC-USDT"
, "BNBSPOT:PAIR-XRP-USDT"
]
, "COINBASE_CLD21": [
"COINBASE:PAIR-ADA-USD"
, "COINBASE:PAIR-BCH-USD"
, "COINBASE:PAIR-BTC-USD"
, "COINBASE:PAIR-DOT-USD"
, "COINBASE:PAIR-ETH-USD"
, "COINBASE:PAIR-LTC-USD"
, "COINBASE:PAIR-SOL-USD"
, "COINBASE:PAIR-XRP-USD"
]
, "COINBASE_CVTTDATA": [
"COINBASE:PAIR-ADA-USD"
, "COINBASE:PAIR-BCH-USD"
, "COINBASE:PAIR-BTC-USD"
, "COINBASE:PAIR-DOT-USD"
, "COINBASE:PAIR-ETH-USD"
, "COINBASE:PAIR-LTC-USD"
, "COINBASE:PAIR-SOL-USD"
, "COINBASE:PAIR-XRP-USD"
]
}
, "db": {
"class": "cvttpy.tools.db.timescale.tsdb_client.TimescaleDbClient"
}
, "flush_interval_sec": 0.5
, "collections": {
"__dummy__": null
, "ALPACA": {
# dynamic instruments names
"tables": {
"trades": "alpaca_iex.md_trades"
, "booktops": "alpaca_iex.md_booktops"
, "booksnaps": "alpaca_iex.md_booksnaps"
}
}
, "BNBSPOT": {
"instruments": {
"PAIR-ADA-USDT": ["TRADES", "BOOK_TOP"]
, "PAIR-BCH-USDT": ["TRADES", "BOOK_TOP"]
, "PAIR-BTC-USDT": ["TRADES", "BOOK_TOP"]
, "PAIR-DOT-USDT": ["TRADES", "BOOK_TOP"]
, "PAIR-ETH-USDT": ["TRADES", "BOOK_TOP"]
, "PAIR-LTC-USDT": ["TRADES", "BOOK_TOP"]
, "PAIR-SOL-USDT": ["TRADES", "BOOK_TOP"]
, "PAIR-USDC-USDT": ["TRADES", "BOOK_TOP"]
, "PAIR-XRP-USDT": ["TRADES", "BOOK_TOP"]
}
, "tables": {
"trades": "bnbspot.md_trades"
, "booktops": "bnbspot.md_booktops"
, "booksnaps": "bnbspot.md_booksnaps"
}
}
, "BNBFUT": {
"instruments": {
"PERP-BTC-USDT": ["TRADES", "BOOK_TOP"]
, "PERP-ETH-USDT": ["TRADES", "BOOK_TOP"]
}
, "tables": {
"trades": "bnbfut.md_trades"
, "booktops": "bnbfut.md_booktops"
, "booksnaps": "bnbfut.md_booksnaps"
}
}
, "COINBASE": {
"instruments": {
"PAIR-ADA-USD": ["TRADES", "BOOK_TOP"]
, "PAIR-BCH-USD": ["TRADES", "BOOK_TOP"]
, "PAIR-BTC-USD": ["TRADES", "BOOK_TOP"]
, "PAIR-DOT-USD": ["TRADES", "BOOK_TOP"]
, "PAIR-ETH-USD": ["TRADES", "BOOK_TOP"]
, "PAIR-LTC-USD": ["TRADES", "BOOK_TOP"]
, "PAIR-SOL-USD": ["TRADES", "BOOK_TOP"]
, "PAIR-XRP-USD": ["TRADES", "BOOK_TOP"]
}
, "tables": {
"trades": "coinbase.md_trades"
, "booktops": "coinbase.md_booktops"
, "booksnaps": "coinbase.md_booksnaps"
}
}
}
}
# ----------------------------------
, "exchanges_settings": {
"*": {
"order_entry": {
"enabled": false
, "streaming": false
},
"mkt_data": {
"enabled": true
, "streaming": true
},
"accounting": {
"enabled": false
, "streaming": false
}
}
}
}

17
market_data/md_recorder/deployment/cvtt-md/.creds Normal file
View File

@ -0,0 +1,17 @@
{
"__dummy__": null
, "TSDB_MD_CLD21": {
"host": "cloud21.cvtt.vpn"
, "port": 5432
, "user": "cvtt"
, "database": "cvtt_md"
, "password": "ICdIh0JnMM7vM7Pf"
}
, "TSDB_MD_CVTTDATA": {
"host": "cvttdata.cvtt.vpn"
, "port": 5432
, "user": "cvtt"
, "database": "cvtt_md"
, "password": "ICdIh0JnMM7vM7Pf"
}
}

28
market_data/md_recorder/deployment/cvtt-md/.env Normal file
View File

@ -0,0 +1,28 @@
# -------- cvtt-md
CVTT_USER=1001:1001
IMAGE_VERSION=0.0.2
CONFIG_FILE=/config/md_recorder.cfg
CREDS_FILE=/config/.creds
# bnbfut-cvttdata
BFCD_CONTAINER_NAME=md_recorder.BNBFUT_CVTTDATA
BFCD_INSTRUMENT_GROUP=BNBFUT_CVTTDATA
BFCD_EXCHANGE_ACCT_NAME=BNBFUT
BFCD_DB_CRED_KEY=TSDB_MD_CVTTDATA
BFCD_LOG_FILE=/logs/%T.MD_REC.${BFCD_INSTRUMENT_GROUP}.log
BFCD_ADD_ARGS=--compress_log
BFCD_ADMIN_PORT=7201
# coinbase-cvttdata
CBCD_CONTAINER_NAME=md_recorder.COINBASE_CVTTDATA
CBCD_INSTRUMENT_GROUP=COINBASE_CVTTDATA
CBCD_EXCHANGE_ACCT_NAME=COINBASE_AT
CBCD_DB_CRED_KEY=TSDB_MD_CVTTDATA
CBCD_LOG_FILE=/logs/%T.MD_REC.${CBCD_INSTRUMENT_GROUP}.log
CBCD_ADD_ARGS=--compress_log
CBCD_ADMIN_PORT=7202

44
market_data/md_recorder/deployment/cvtt-md/docker-compose.yml Normal file
View File

@ -0,0 +1,44 @@
# CVTT Microservices
services:
bfcd_md_recorder:
image: cloud21.cvtt.vpn:5500/md_recorder:${IMAGE_VERSION}
container_name: ${BFCD_CONTAINER_NAME}
user: ${CVTT_USER:-1001:1001}
pull_policy: always
restart: unless-stopped
environment:
CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg}
CREDS_FILE: ${CREDS_FILE}
EXCHANGE_ACCT_NAME: ${BFCD_EXCHANGE_ACCT_NAME}
INSTRUMENT_GROUP: ${BFCD_INSTRUMENT_GROUP}
DB_CRED_KEY: ${BFCD_DB_CRED_KEY}
LOG_FILE: ${BFCD_LOG_FILE}
ADD_ARGS: ${BFCD_ADD_ARGS:-}
volumes:
- ./config:/config
- ./logs:/logs
ports:
- ${BFCD_ADMIN_PORT}:7200
cbcd_md_recorder:
image: cloud21.cvtt.vpn:5500/md_recorder:${IMAGE_VERSION}
container_name: ${CBCD_CONTAINER_NAME}
user: ${CVTT_USER:-1001:1001}
pull_policy: always
restart: unless-stopped
environment:
CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg}
CREDS_FILE: ${CREDS_FILE}
EXCHANGE_ACCT_NAME: ${CBCD_EXCHANGE_ACCT_NAME}
INSTRUMENT_GROUP: ${CBCD_INSTRUMENT_GROUP}
DB_CRED_KEY: ${CBCD_DB_CRED_KEY}
LOG_FILE: ${CBCD_LOG_FILE}
ADD_ARGS: ${CBCD_ADD_ARGS:-}
volumes:
- ./config:/config
- ./logs:/logs
ports:
- ${CBCD_ADMIN_PORT}:7200

206
market_data/md_recorder/deployment/cvtt-md/md_recorder.cfg Normal file
View File

@ -0,0 +1,206 @@
{
"refdata": {
"assets": @inc=http://@env{CONFIG_SERVICE}/refdata/assets
, "instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/instruments
, "exchange_instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/exchange_instruments
, "dynamic_instrument_exchanges": ["ALPACA"]
, "exchanges": {
"COINBASE_AT": {
"exchange_id": "COINBASE"
, "name": "Coinbase Advanced Trade"
, "type": "SPOT"
, "mktdata" : {
# ****** NEW VAlUE
# "class": "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMultiWsFeed"
"class": "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMdFeed"
# ****** NEW VAlUE
, "reconnect_secs": 2
, "max_connections": 10
, "price_throttle": true
, "ws_url": ["WSS", "wss://ws-feed.exchange.coinbase.com"]
}
}
, "BNBSPOT": {
"exchange_id": "BNBSPOT"
, "name": "Binance Spot"
, "type": "SPOT"
, "mktdata" : {
# -------- NEW VALUE
# "class": "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMultiWsFeed"
"class": "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMktDataFeed"
# -------- NEW VALUE
, "max_connections": 10
, "reconnect_secs": 2.0
, "ws_url": ["WSS", "wss://stream.binance.com/ws"]
, "REST": {
"depth_snapshot_url": ["GET", "https://api.binance.com/api/v3/depth"],
}
}
}
, "BNBFUT": {
"exchange_id": "BNBFUT"
, "name": "Binance Futures USD-M"
, "type": "FUTURES"
, "mktdata" : {
# -------- NEW VALUE
# "class": "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMultiWsFeed"
"class": "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMktDataFeed"
# -------- NEW VALUE
, "max_connections": 10
, "reconnect_secs": 2.0
, "ws_url": ["WSS", "wss://fstream.binance.com/ws"]
, "REST": {
# https://fapi.binance.com/fapi/v1/depth?symbol=BNBBTC&limit=1000
"depth_snapshot_url": ["GET", "https://fapi.binance.com/fapi/v1/depth"],
}
}
}
}
}
, "comm": @inc=http://@env{CONFIG_SERVICE}/apps/common/comm
, "md_recorder": {
"instrument_groups": {
"BNBFUT_CLD21": [
"BNBFUT:PERP-BTC-USDT"
, "BNBFUT:PERP-ETH-USDT"
]
"BNBFUT_CVTTDATA": [
"BNBFUT:PERP-BTC-USDT"
, "BNBFUT:PERP-ETH-USDT"
]
, "BNBSPOT_CLD21": [
"BNBSPOT:PAIR-ADA-USDT"
, "BNBSPOT:PAIR-BCH-USDT"
, "BNBSPOT:PAIR-BTC-USDT"
, "BNBSPOT:PAIR-DOT-USDT"
, "BNBSPOT:PAIR-ETH-USDT"
, "BNBSPOT:PAIR-LTC-USDT"
, "BNBSPOT:PAIR-SOL-USDT"
, "BNBSPOT:PAIR-USDC-USDT"
, "BNBSPOT:PAIR-XRP-USDT"
]
, "BNBSPOT_CVTTDATA": [
"BNBSPOT:PAIR-ADA-USDT"
, "BNBSPOT:PAIR-BCH-USDT"
, "BNBSPOT:PAIR-BTC-USDT"
, "BNBSPOT:PAIR-DOT-USDT"
, "BNBSPOT:PAIR-ETH-USDT"
, "BNBSPOT:PAIR-LTC-USDT"
, "BNBSPOT:PAIR-SOL-USDT"
, "BNBSPOT:PAIR-USDC-USDT"
, "BNBSPOT:PAIR-XRP-USDT"
]
, "COINBASE_CLD21": [
"COINBASE:PAIR-ADA-USD"
, "COINBASE:PAIR-BCH-USD"
, "COINBASE:PAIR-BTC-USD"
, "COINBASE:PAIR-DOT-USD"
, "COINBASE:PAIR-ETH-USD"
, "COINBASE:PAIR-LTC-USD"
, "COINBASE:PAIR-SOL-USD"
, "COINBASE:PAIR-XRP-USD"
]
, "COINBASE_CVTTDATA": [
"COINBASE:PAIR-ADA-USD"
, "COINBASE:PAIR-BCH-USD"
, "COINBASE:PAIR-BTC-USD"
, "COINBASE:PAIR-DOT-USD"
, "COINBASE:PAIR-ETH-USD"
, "COINBASE:PAIR-LTC-USD"
, "COINBASE:PAIR-SOL-USD"
, "COINBASE:PAIR-XRP-USD"
]
}
, "db": {
"class": "cvttpy.tools.db.timescale.tsdb_client.TimescaleDbClient"
}
, "flush_interval_sec": 0.5
, "collections": {
"__dummy__": null
, "ALPACA": {
# dynamic instruments names
"tables": {
"trades": "alpaca_iex.md_trades"
, "booktops": "alpaca_iex.md_booktops"
, "booksnaps": "alpaca_iex.md_booksnaps"
}
}
, "BNBSPOT": {
"instruments": {
"PAIR-ADA-USDT": ["TRADES", "BOOK_TOP"]
, "PAIR-BCH-USDT": ["TRADES", "BOOK_TOP"]
, "PAIR-BTC-USDT": ["TRADES", "BOOK_TOP"]
, "PAIR-DOT-USDT": ["TRADES", "BOOK_TOP"]
, "PAIR-ETH-USDT": ["TRADES", "BOOK_TOP"]
, "PAIR-LTC-USDT": ["TRADES", "BOOK_TOP"]
, "PAIR-SOL-USDT": ["TRADES", "BOOK_TOP"]
, "PAIR-USDC-USDT": ["TRADES", "BOOK_TOP"]
, "PAIR-XRP-USDT": ["TRADES", "BOOK_TOP"]
}
, "tables": {
"trades": "bnbspot.md_trades"
, "booktops": "bnbspot.md_booktops"
, "booksnaps": "bnbspot.md_booksnaps"
}
}
, "BNBFUT": {
"instruments": {
"PERP-BTC-USDT": ["TRADES", "BOOK_TOP"]
, "PERP-ETH-USDT": ["TRADES", "BOOK_TOP"]
}
, "tables": {
"trades": "bnbfut.md_trades"
, "booktops": "bnbfut.md_booktops"
, "booksnaps": "bnbfut.md_booksnaps"
}
}
, "COINBASE": {
"instruments": {
"PAIR-ADA-USD": ["TRADES", "BOOK_TOP"]
, "PAIR-BCH-USD": ["TRADES", "BOOK_TOP"]
, "PAIR-BTC-USD": ["TRADES", "BOOK_TOP"]
, "PAIR-DOT-USD": ["TRADES", "BOOK_TOP"]
, "PAIR-ETH-USD": ["TRADES", "BOOK_TOP"]
, "PAIR-LTC-USD": ["TRADES", "BOOK_TOP"]
, "PAIR-SOL-USD": ["TRADES", "BOOK_TOP"]
, "PAIR-XRP-USD": ["TRADES", "BOOK_TOP"]
}
, "tables": {
"trades": "coinbase.md_trades"
, "booktops": "coinbase.md_booktops"
, "booksnaps": "coinbase.md_booksnaps"
}
}
}
}
# ----------------------------------
, "exchanges_settings": {
"*": {
"order_entry": {
"enabled": false
, "streaming": false
},
"mkt_data": {
"enabled": true
, "streaming": true
},
"accounting": {
"enabled": false
, "streaming": false
}
}
}
}

22
market_data/md_recorder/entrypoint.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
# runs in container
Cmd="python3.12"
Cmd+=" cvttpy/apps/md/md_recorder.py"
Cmd+=" --config=${CONFIG_FILE:-/config/cvtt_musvc.cfg}"
Cmd+=" --credentials_file=${CREDS_FILE:-/config/.creds}"
Cmd="${Cmd} --instrument_group=${INSTRUMENT_GROUP}"
Cmd="${Cmd} --exch_acct_cfgname=${EXCHANGE_ACCT_NAME}"
Cmd="${Cmd} --db_credentials_key=${DB_CRED_KEY}"
Cmd="${Cmd} --admin_port=7200"
Cmd+=" --log_file=${LOG_FILE:-/logs/%T.MD_REC.${INSTRUMENT_GROUP}.log}"
Cmd+=" ${ADD_ARGS}"
echo ${Cmd}
exec ${Cmd}

1
market_data/md_recorder/requirements.txt
View File

@ -5,3 +5,4 @@ hjson>=3.1.0
pandas>=1.5.3
sortedcontainers>=2.4.0
redis>=5.0.8
debugpy>=1.8.13

69
market_data/md_recorder/run_md_recorder.sh
View File

@ -1,69 +0,0 @@
#!/bin/bash
# runs in container
ValidJobs=('BNBFUT_CLD21' 'BNBSPOT_CLD21' 'COINBASE_CLD21' 'BNBFUT_CVTTDATA' 'BNBSPOT_CVTTDATA' 'COINBASE_CVTTDATA')
# runs on host to start container
usage() {
echo "Usage: $0 <job_name; one of (${ValidJobs[@]})>"
exit 1
}
is_valid() {
local job=$1
for valid_job in "${ValidJobs[@]}";
do
# echo "job=$job valid_job=$valid_job"
if [[ "${job}" == "${valid_job}" ]]; then
return 0
fi
done
return 1
}
job=${1}
if ! is_valid "${job}"; then
usage
fi
ConfigServer=cloud16.cvtt.vpn:6789
Cmd="python3.12"
Cmd="${Cmd} cvttpy/apps/md/md_recorder.py"
Cmd="${Cmd} --config=http://${ConfigServer}/apps/md_recorder"
Cmd="${Cmd} --credentials_file=/.creds"
Cmd="${Cmd} --compress_log"
Cmd="${Cmd} --instrument_group=${job}"
Cmd="${Cmd} --log_file=/logs/%T.MD_REC.${job}.log"
if [ "${job}" == "BNBFUT_CLD21" ] ; then
Cmd="${Cmd} --exch_acct_cfgname=BNBFUT"
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
Cmd="${Cmd} --admin_port=7201"
elif [ "${job}" == "COINBASE_CLD21" ] ; then
Cmd="${Cmd} --exch_acct_cfgname=COINBASE_AT"
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
Cmd="${Cmd} --admin_port=7202"
elif [ "${job}" == "BNBSPOT_CLD21" ] ; then
Cmd="${Cmd} --exch_acct_cfgname=BNBSPOT"
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
Cmd="${Cmd} --admin_port=7203"
elif [ "${job}" == "BNBSPOT_CVTTDATA" ] ; then
Cmd="${Cmd} --exch_acct_cfgname=BNBSPOT"
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
Cmd="${Cmd} --admin_port=7204"
elif [ "${job}" == "BNBFUT_CVTTDATA" ] ; then
Cmd="${Cmd} --exch_acct_cfgname=BNBFUT"
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
Cmd="${Cmd} --admin_port=7205"
elif [ "${job}" == "COINBASE_CVTTDATA" ] ; then
Cmd="${Cmd} --exch_acct_cfgname=COINBASE_AT"
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
Cmd="${Cmd} --admin_port=7206"
else
echo "Unrecognized JOB: ${job}"
exit 1
fi
echo ${Cmd}
eval ${Cmd}

2
microservices/mu_svc_functions.sh
View File

@ -17,7 +17,7 @@ get_project_version() {
echo "Checking ${user}@${host} for ${rel_dir}/${Project}/${Version} ..."
if ssh -q -p ${port} ${user}@${host} "test -d ${rel_dir}/${Project}/${Version}"
then
echo "Version ${Version} found..."
echo "Getting Version \"${Version}\"..."
rsync_cmd="rsync -ahv -e \"ssh -p ${port}\""
rsync_cmd="${rsync_cmd} ${user}@${host}:${rel_dir}/${Project}/${Version}/"
rsync_cmd="${rsync_cmd} ./"

76
mu_svc_functions.sh Executable file
View File

@ -0,0 +1,76 @@
#!/bin/bash
get_project_version() {
Project=${1}
Version=${2}
host="cloud21.cvtt.vpn"
port="22"
user="cvttdist"
rel_dir="/home/cvttdist/software/cvtt2"
if [ "${Version}" == "latest" ]; then
echo "Checking for latest version of ${Project} on ${user}@${host}:${rel_dir}"
Version=$(ssh -q -p ${port} ${user}@${host} "ls -tr ${rel_dir}/${Project} | tail -1" )
echo "Latest version is ${Version}"
fi
echo "Checking ${user}@${host} for ${rel_dir}/${Project}/${Version} ..."
if ssh -q -p ${port} ${user}@${host} "test -d ${rel_dir}/${Project}/${Version}"
then
echo "Getting Version \"${Version}\"..."
rsync_cmd="rsync -ahv -e \"ssh -p ${port}\""
rsync_cmd="${rsync_cmd} ${user}@${host}:${rel_dir}/${Project}/${Version}/"
rsync_cmd="${rsync_cmd} ./"
echo ${rsync_cmd}
eval ${rsync_cmd}
status=$?
if [ ${status} -eq 0 ]
then
echo "Loading successful..."
return 0
else
return 1
fi
else
echo "Not Found ${rel_dir}/${Project}/${Version} on ${user}@${host}"
return 1
fi
}
build_docker_image() {
ImageName=${1}
RegistryService=${2}
Version=${3}
Project=${4}
ProjectVersion=${5}
Cmd="docker build"
if [ "${ProjectVersion}" != "" ]; then
Cmd+=" --label ${Project}=\"${ProjectVersion}\""
fi
Cmd+=" -t ${ImageName}"
Cmd+=" -t ${ImageName}:latest"
Cmd+=" -t ${ImageName}:${Version}"
Cmd+=" -f Dockerfile"
Cmd+=" ."
echo ${Cmd}
eval ${Cmd} || exit
Cmd="docker tag ${ImageName}:latest ${RegistryService}/${ImageName}:latest"
echo ${Cmd}
eval ${Cmd} || exit
Cmd="docker tag ${ImageName}:${Version} ${RegistryService}/${ImageName}:${Version}"
echo ${Cmd}
eval ${Cmd} || exit
Cmd="docker push ${RegistryService}/${ImageName}:latest"
echo ${Cmd}
eval ${Cmd} || exit
Cmd="docker push ${RegistryService}/${ImageName}:${Version}"
echo ${Cmd}
eval ${Cmd} || exit
}