From f09865d3a30337e44cb11397a347c0bf210af4f5 Mon Sep 17 00:00:00 2001 From: Oleg Sheynin Date: Wed, 9 Apr 2025 17:55:33 -0400 Subject: [PATCH] docker compose based md_recorder --- VERSION | 2 +- market_data/md_recorder/Dockerfile | 22 +- market_data/md_recorder/build.sh | 69 ++++++ .../{ => deployment/cloud24}/.creds | 0 .../md_recorder/deployment/cloud24/.env | 52 +++++ .../deployment/cloud24/docker-compose.yml | 86 ++++++++ .../deployment/cloud24/md_recorder.cfg | 206 ++++++++++++++++++ .../md_recorder/deployment/cvtt-md/.creds | 17 ++ .../md_recorder/deployment/cvtt-md/.env | 28 +++ .../deployment/cvtt-md/docker-compose.yml | 44 ++++ .../deployment/cvtt-md/md_recorder.cfg | 206 ++++++++++++++++++ market_data/md_recorder/entrypoint.sh | 22 ++ market_data/md_recorder/requirements.txt | 1 + market_data/md_recorder/run_md_recorder.sh | 69 ------ microservices/mu_svc_functions.sh | 2 +- mu_svc_functions.sh | 76 +++++++ 16 files changed, 819 insertions(+), 83 deletions(-) create mode 100755 market_data/md_recorder/build.sh rename market_data/md_recorder/{ => deployment/cloud24}/.creds (100%) create mode 100644 market_data/md_recorder/deployment/cloud24/.env create mode 100644 market_data/md_recorder/deployment/cloud24/docker-compose.yml create mode 100644 market_data/md_recorder/deployment/cloud24/md_recorder.cfg create mode 100644 market_data/md_recorder/deployment/cvtt-md/.creds create mode 100644 market_data/md_recorder/deployment/cvtt-md/.env create mode 100644 market_data/md_recorder/deployment/cvtt-md/docker-compose.yml create mode 100644 market_data/md_recorder/deployment/cvtt-md/md_recorder.cfg create mode 100755 market_data/md_recorder/entrypoint.sh delete mode 100755 market_data/md_recorder/run_md_recorder.sh create mode 100755 mu_svc_functions.sh diff --git a/VERSION b/VERSION index 80db2c2..3a9e6a3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.6,improved build.sh for microservices \ No newline at end of file +0.2.7,new md_recorder build \ No newline at end of file diff --git a/market_data/md_recorder/Dockerfile b/market_data/md_recorder/Dockerfile index e15a591..e15e67f 100644 --- a/market_data/md_recorder/Dockerfile +++ b/market_data/md_recorder/Dockerfile @@ -1,27 +1,25 @@ FROM python:3.12-slim -ARG FROM_DIR=docker_dev/market_data/md_recorder - +# libpq needed for Timescale RUN apt-get update && apt-get install -y \ apt-utils \ libpq-dev \ build-essential -COPY ${FROM_DIR}/requirements.txt / +COPY requirements.txt / + RUN pip install --upgrade pip --root-user-action=ignore RUN pip install -r /requirements.txt --root-user-action=ignore COPY cvttpy /cvttpy -COPY ${FROM_DIR}/run_md_recorder.sh /run_md_recorder.sh +RUN mkdir -p /logs /config -# Shared Volumes -RUN mkdir -p /app/data -RUN mkdir /logs -RUN chmod +x /run_md_recorder.sh - -WORKDIR / SHELL ["/bin/bash", "-c"] -ENV PYTHONPATH=/ -ENTRYPOINT [ "/run_md_recorder.sh" ] +ENV PYTHONPATH=/ + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/market_data/md_recorder/build.sh b/market_data/md_recorder/build.sh new file mode 100755 index 0000000..6939ba5 --- /dev/null +++ b/market_data/md_recorder/build.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# --- Settings +RegistryService=cloud21.cvtt.vpn:5500 +ImageName=md_recorder +BashFunctionCode=$(realpath $(dirname ${0}))/../../mu_svc_functions.sh +# --- Settings + +usage() { + echo "Usage: $0 -V -L :" + exit 1 +} + + +# ---------------- cmdline +while getopts "V:L:" opt; do + case ${opt} in + V ) + ImageVersion=$OPTARG + ;; + L ) + Library=$OPTARG + ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + usage + ;; + : ) + echo "Option -$OPTARG requires an argument." >&2 + usage + ;; + esac +done +# ---------------- cmdline + +if [ "${ImageVersion}" == "" ] +then + usage +fi + +Project= +ProjectVersion= +if [ "${Library}" != "" ] ; then + IFS=':' read -ra parts <<< "${Library}" + Project=${parts[0]} + ProjectVersion=${parts[1]} +fi + +DockerDir=$(realpath $(dirname ${0})) + +cd ${DockerDir} +source ${BashFunctionCode} + +if [ "${Project}" != "" ]; then + get_project_version ${Project} ${ProjectVersion} +fi +function cleanup { + cd ${DockerDir} + rm -rf cvttpy +} +trap cleanup EXIT + +build_docker_image ${ImageName} ${RegistryService} ${ImageVersion} ${Project} ${ProjectVersion} + + +# echo "**** D E B U G E X I T" && exit +echo "***** ${0} D O N E" + + diff --git a/market_data/md_recorder/.creds b/market_data/md_recorder/deployment/cloud24/.creds similarity index 100% rename from market_data/md_recorder/.creds rename to market_data/md_recorder/deployment/cloud24/.creds diff --git a/market_data/md_recorder/deployment/cloud24/.env b/market_data/md_recorder/deployment/cloud24/.env new file mode 100644 index 0000000..6d8c1dc --- /dev/null +++ b/market_data/md_recorder/deployment/cloud24/.env @@ -0,0 +1,52 @@ +# ----- cloud24 settings +CVTT_USER=1001:1001 +IMAGE_VERSION=0.0.2 + +CONFIG_FILE=/config/md_recorder.cfg +CREDS_FILE=/config/.creds + +# bnspot-cvttdata +BSCD_CONTAINER_NAME=md_recorder.BNBSPOT_CVTTDATA +BSCD_INSTRUMENT_GROUP=BNBSPOT_CVTTDATA +BSCD_EXCHANGE_ACCT_NAME=BNBSPOT +BSCD_DB_CRED_KEY=TSDB_MD_CVTTDATA + +BSCD_LOG_FILE=/logs/%T.MD_REC.${BSCD_INSTRUMENT_GROUP}.log +BSCD_ADD_ARGS=--compress_log + +BSCD_ADMIN_PORT=7201 + +# bnspot-cloud21 +BSCLD21_CONTAINER_NAME=md_recorder.BNBSPOT_CLD21 +BSCLD21_INSTRUMENT_GROUP=BNBSPOT_CLD21 +BSCLD21_EXCHANGE_ACCT_NAME=BNBSPOT +BSCLD21_DB_CRED_KEY=TSDB_MD_CLD21 + +BSCLD21_LOG_FILE=/logs/%T.MD_REC.${BSCLD21_INSTRUMENT_GROUP}.log +BSCLD21_ADD_ARGS=--compress_log + +BSCLD21_ADMIN_PORT=7202 + +# bnbfut-cloud21 +BFCLD21_CONTAINER_NAME=md_recorder.BNBFUT_CLD21 +BFCLD21_INSTRUMENT_GROUP=BNBFUT_CLD21 +BFCLD21_EXCHANGE_ACCT_NAME=BNBFUT +BFCLD21_DB_CRED_KEY=TSDB_MD_CLD21 + +BFCLD21_LOG_FILE=/logs/%T.MD_REC.${BFCLD21_INSTRUMENT_GROUP}.log +BFCLD21_ADD_ARGS=--compress_log + +BFCLD21_ADMIN_PORT=7203 + +# coinbase-cloud21 +CBCLD21_CONTAINER_NAME=md_recorder.COINBASE_CLD21 +CBCLD21_INSTRUMENT_GROUP=COINBASE_CLD21 +CBCLD21_EXCHANGE_ACCT_NAME=COINBASE_AT +CBCLD21_DB_CRED_KEY=TSDB_MD_CLD21 + +CBCLD21_LOG_FILE=/logs/%T.MD_REC.${CBCLD21_INSTRUMENT_GROUP}.log +CBCLD21_ADD_ARGS=--compress_log + +CBCLD21_ADMIN_PORT=7204 + + diff --git a/market_data/md_recorder/deployment/cloud24/docker-compose.yml b/market_data/md_recorder/deployment/cloud24/docker-compose.yml new file mode 100644 index 0000000..9e948be --- /dev/null +++ b/market_data/md_recorder/deployment/cloud24/docker-compose.yml @@ -0,0 +1,86 @@ +# CVTT Microservices +services: + bfcld21_md_recorder: + image: cloud21.cvtt.vpn:5500/md_recorder:${IMAGE_VERSION} + container_name: ${BFCLD21_CONTAINER_NAME} + user: ${CVTT_USER:-1001:1001} + pull_policy: always + restart: unless-stopped + environment: + CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789} + CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg} + CREDS_FILE: ${CREDS_FILE} + EXCHANGE_ACCT_NAME: ${BFCLD21_EXCHANGE_ACCT_NAME} + INSTRUMENT_GROUP: ${BFCLD21_INSTRUMENT_GROUP} + DB_CRED_KEY: ${BFCLD21_DB_CRED_KEY} + LOG_FILE: ${BFCLD21_LOG_FILE} + ADD_ARGS: ${BFCLD21_ADD_ARGS:-} + volumes: + - ./config:/config + - ./logs:/logs + ports: + - ${BFCLD21_ADMIN_PORT}:7200 + + + cbcld21_md_recorder: + image: cloud21.cvtt.vpn:5500/md_recorder:${IMAGE_VERSION} + container_name: ${CBCLD21_CONTAINER_NAME} + user: ${CVTT_USER:-1001:1001} + pull_policy: always + restart: unless-stopped + environment: + CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789} + CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg} + CREDS_FILE: ${CREDS_FILE} + EXCHANGE_ACCT_NAME: ${CBCLD21_EXCHANGE_ACCT_NAME} + INSTRUMENT_GROUP: ${CBCLD21_INSTRUMENT_GROUP} + DB_CRED_KEY: ${CBCLD21_DB_CRED_KEY} + LOG_FILE: ${CBCLD21_LOG_FILE} + ADD_ARGS: ${CBCLD21_ADD_ARGS:-} + volumes: + - ./config:/config + - ./logs:/logs + ports: + - ${CBCLD21_ADMIN_PORT}:7200 + + bscld21_md_recorder: + image: cloud21.cvtt.vpn:5500/md_recorder:${IMAGE_VERSION} + container_name: ${BSCLD21_CONTAINER_NAME} + user: ${CVTT_USER:-1001:1001} + pull_policy: always + restart: unless-stopped + environment: + CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789} + CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg} + CREDS_FILE: ${CREDS_FILE} + EXCHANGE_ACCT_NAME: ${BSCLD21_EXCHANGE_ACCT_NAME} + INSTRUMENT_GROUP: ${BSCLD21_INSTRUMENT_GROUP} + DB_CRED_KEY: ${BSCLD21_DB_CRED_KEY} + LOG_FILE: ${BSCLD21_LOG_FILE} + ADD_ARGS: ${BSCLD21_ADD_ARGS:-} + volumes: + - ./config:/config + - ./logs:/logs + ports: + - ${BSCLD21_ADMIN_PORT}:7200 + + bscd_md_recorder: + image: cloud21.cvtt.vpn:5500/md_recorder:${IMAGE_VERSION} + container_name: ${BSCD_CONTAINER_NAME} + user: ${CVTT_USER:-1001:1001} + pull_policy: always + restart: unless-stopped + environment: + CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789} + CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg} + CREDS_FILE: ${CREDS_FILE} + EXCHANGE_ACCT_NAME: ${BSCD_EXCHANGE_ACCT_NAME} + INSTRUMENT_GROUP: ${BSCD_INSTRUMENT_GROUP} + DB_CRED_KEY: ${BSCD_DB_CRED_KEY} + LOG_FILE: ${BSCD_LOG_FILE} + ADD_ARGS: ${BSCD_ADD_ARGS:-} + volumes: + - ./config:/config + - ./logs:/logs + ports: + - ${BSCD_ADMIN_PORT}:7200 diff --git a/market_data/md_recorder/deployment/cloud24/md_recorder.cfg b/market_data/md_recorder/deployment/cloud24/md_recorder.cfg new file mode 100644 index 0000000..89c7ba6 --- /dev/null +++ b/market_data/md_recorder/deployment/cloud24/md_recorder.cfg @@ -0,0 +1,206 @@ +{ + "refdata": { + "assets": @inc=http://@env{CONFIG_SERVICE}/refdata/assets + , "instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/instruments + , "exchange_instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/exchange_instruments + , "dynamic_instrument_exchanges": ["ALPACA"] + , "exchanges": { + "COINBASE_AT": { + "exchange_id": "COINBASE" + , "name": "Coinbase Advanced Trade" + , "type": "SPOT" + + , "mktdata" : { + # ****** NEW VAlUE + # "class": "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMultiWsFeed" + "class": "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMdFeed" + # ****** NEW VAlUE + + , "reconnect_secs": 2 + , "max_connections": 10 + , "price_throttle": true + + , "ws_url": ["WSS", "wss://ws-feed.exchange.coinbase.com"] + } + } + , "BNBSPOT": { + "exchange_id": "BNBSPOT" + , "name": "Binance Spot" + , "type": "SPOT" + + , "mktdata" : { + # -------- NEW VALUE + # "class": "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMultiWsFeed" + "class": "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMktDataFeed" + # -------- NEW VALUE + , "max_connections": 10 + , "reconnect_secs": 2.0 + + + , "ws_url": ["WSS", "wss://stream.binance.com/ws"] + , "REST": { + "depth_snapshot_url": ["GET", "https://api.binance.com/api/v3/depth"], + } + } + } + , "BNBFUT": { + "exchange_id": "BNBFUT" + , "name": "Binance Futures USD-M" + , "type": "FUTURES" + + , "mktdata" : { + # -------- NEW VALUE + # "class": "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMultiWsFeed" + "class": "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMktDataFeed" + # -------- NEW VALUE + , "max_connections": 10 + , "reconnect_secs": 2.0 + + , "ws_url": ["WSS", "wss://fstream.binance.com/ws"] + + , "REST": { + # https://fapi.binance.com/fapi/v1/depth?symbol=BNBBTC&limit=1000 + "depth_snapshot_url": ["GET", "https://fapi.binance.com/fapi/v1/depth"], + } + } + } + } + } + , "comm": @inc=http://@env{CONFIG_SERVICE}/apps/common/comm + + , "md_recorder": { + "instrument_groups": { + "BNBFUT_CLD21": [ + "BNBFUT:PERP-BTC-USDT" + , "BNBFUT:PERP-ETH-USDT" + ] + "BNBFUT_CVTTDATA": [ + "BNBFUT:PERP-BTC-USDT" + , "BNBFUT:PERP-ETH-USDT" + ] + , "BNBSPOT_CLD21": [ + "BNBSPOT:PAIR-ADA-USDT" + , "BNBSPOT:PAIR-BCH-USDT" + , "BNBSPOT:PAIR-BTC-USDT" + , "BNBSPOT:PAIR-DOT-USDT" + , "BNBSPOT:PAIR-ETH-USDT" + , "BNBSPOT:PAIR-LTC-USDT" + , "BNBSPOT:PAIR-SOL-USDT" + , "BNBSPOT:PAIR-USDC-USDT" + , "BNBSPOT:PAIR-XRP-USDT" + ] + , "BNBSPOT_CVTTDATA": [ + "BNBSPOT:PAIR-ADA-USDT" + , "BNBSPOT:PAIR-BCH-USDT" + , "BNBSPOT:PAIR-BTC-USDT" + , "BNBSPOT:PAIR-DOT-USDT" + , "BNBSPOT:PAIR-ETH-USDT" + , "BNBSPOT:PAIR-LTC-USDT" + , "BNBSPOT:PAIR-SOL-USDT" + , "BNBSPOT:PAIR-USDC-USDT" + , "BNBSPOT:PAIR-XRP-USDT" + ] + , "COINBASE_CLD21": [ + "COINBASE:PAIR-ADA-USD" + , "COINBASE:PAIR-BCH-USD" + , "COINBASE:PAIR-BTC-USD" + , "COINBASE:PAIR-DOT-USD" + , "COINBASE:PAIR-ETH-USD" + , "COINBASE:PAIR-LTC-USD" + , "COINBASE:PAIR-SOL-USD" + , "COINBASE:PAIR-XRP-USD" + ] + , "COINBASE_CVTTDATA": [ + "COINBASE:PAIR-ADA-USD" + , "COINBASE:PAIR-BCH-USD" + , "COINBASE:PAIR-BTC-USD" + , "COINBASE:PAIR-DOT-USD" + , "COINBASE:PAIR-ETH-USD" + , "COINBASE:PAIR-LTC-USD" + , "COINBASE:PAIR-SOL-USD" + , "COINBASE:PAIR-XRP-USD" + ] + } + , "db": { + "class": "cvttpy.tools.db.timescale.tsdb_client.TimescaleDbClient" + } + , "flush_interval_sec": 0.5 + + , "collections": { + "__dummy__": null + , "ALPACA": { + # dynamic instruments names + "tables": { + "trades": "alpaca_iex.md_trades" + , "booktops": "alpaca_iex.md_booktops" + , "booksnaps": "alpaca_iex.md_booksnaps" + } + } + , "BNBSPOT": { + "instruments": { + "PAIR-ADA-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-BCH-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-BTC-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-DOT-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-ETH-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-LTC-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-SOL-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-USDC-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-XRP-USDT": ["TRADES", "BOOK_TOP"] + } + , "tables": { + "trades": "bnbspot.md_trades" + , "booktops": "bnbspot.md_booktops" + , "booksnaps": "bnbspot.md_booksnaps" + } + } + , "BNBFUT": { + "instruments": { + "PERP-BTC-USDT": ["TRADES", "BOOK_TOP"] + , "PERP-ETH-USDT": ["TRADES", "BOOK_TOP"] + } + , "tables": { + "trades": "bnbfut.md_trades" + , "booktops": "bnbfut.md_booktops" + , "booksnaps": "bnbfut.md_booksnaps" + } + } + , "COINBASE": { + "instruments": { + "PAIR-ADA-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-BCH-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-BTC-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-DOT-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-ETH-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-LTC-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-SOL-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-XRP-USD": ["TRADES", "BOOK_TOP"] + } + , "tables": { + "trades": "coinbase.md_trades" + , "booktops": "coinbase.md_booktops" + , "booksnaps": "coinbase.md_booksnaps" + } + } + } + } + + # ---------------------------------- + , "exchanges_settings": { + "*": { + "order_entry": { + "enabled": false + , "streaming": false + }, + "mkt_data": { + "enabled": true + , "streaming": true + }, + "accounting": { + "enabled": false + , "streaming": false + } + } + } +} + diff --git a/market_data/md_recorder/deployment/cvtt-md/.creds b/market_data/md_recorder/deployment/cvtt-md/.creds new file mode 100644 index 0000000..1301a78 --- /dev/null +++ b/market_data/md_recorder/deployment/cvtt-md/.creds @@ -0,0 +1,17 @@ +{ + "__dummy__": null + , "TSDB_MD_CLD21": { + "host": "cloud21.cvtt.vpn" + , "port": 5432 + , "user": "cvtt" + , "database": "cvtt_md" + , "password": "ICdIh0JnMM7vM7Pf" + } + , "TSDB_MD_CVTTDATA": { + "host": "cvttdata.cvtt.vpn" + , "port": 5432 + , "user": "cvtt" + , "database": "cvtt_md" + , "password": "ICdIh0JnMM7vM7Pf" + } +} diff --git a/market_data/md_recorder/deployment/cvtt-md/.env b/market_data/md_recorder/deployment/cvtt-md/.env new file mode 100644 index 0000000..0e96df8 --- /dev/null +++ b/market_data/md_recorder/deployment/cvtt-md/.env @@ -0,0 +1,28 @@ +# -------- cvtt-md +CVTT_USER=1001:1001 +IMAGE_VERSION=0.0.2 + +CONFIG_FILE=/config/md_recorder.cfg +CREDS_FILE=/config/.creds + +# bnbfut-cvttdata +BFCD_CONTAINER_NAME=md_recorder.BNBFUT_CVTTDATA +BFCD_INSTRUMENT_GROUP=BNBFUT_CVTTDATA +BFCD_EXCHANGE_ACCT_NAME=BNBFUT +BFCD_DB_CRED_KEY=TSDB_MD_CVTTDATA + +BFCD_LOG_FILE=/logs/%T.MD_REC.${BFCD_INSTRUMENT_GROUP}.log +BFCD_ADD_ARGS=--compress_log + +BFCD_ADMIN_PORT=7201 + +# coinbase-cvttdata +CBCD_CONTAINER_NAME=md_recorder.COINBASE_CVTTDATA +CBCD_INSTRUMENT_GROUP=COINBASE_CVTTDATA +CBCD_EXCHANGE_ACCT_NAME=COINBASE_AT +CBCD_DB_CRED_KEY=TSDB_MD_CVTTDATA + +CBCD_LOG_FILE=/logs/%T.MD_REC.${CBCD_INSTRUMENT_GROUP}.log +CBCD_ADD_ARGS=--compress_log + +CBCD_ADMIN_PORT=7202 diff --git a/market_data/md_recorder/deployment/cvtt-md/docker-compose.yml b/market_data/md_recorder/deployment/cvtt-md/docker-compose.yml new file mode 100644 index 0000000..2d1991f --- /dev/null +++ b/market_data/md_recorder/deployment/cvtt-md/docker-compose.yml @@ -0,0 +1,44 @@ +# CVTT Microservices +services: + bfcd_md_recorder: + image: cloud21.cvtt.vpn:5500/md_recorder:${IMAGE_VERSION} + container_name: ${BFCD_CONTAINER_NAME} + user: ${CVTT_USER:-1001:1001} + pull_policy: always + restart: unless-stopped + environment: + CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789} + CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg} + CREDS_FILE: ${CREDS_FILE} + EXCHANGE_ACCT_NAME: ${BFCD_EXCHANGE_ACCT_NAME} + INSTRUMENT_GROUP: ${BFCD_INSTRUMENT_GROUP} + DB_CRED_KEY: ${BFCD_DB_CRED_KEY} + LOG_FILE: ${BFCD_LOG_FILE} + ADD_ARGS: ${BFCD_ADD_ARGS:-} + volumes: + - ./config:/config + - ./logs:/logs + ports: + - ${BFCD_ADMIN_PORT}:7200 + + + cbcd_md_recorder: + image: cloud21.cvtt.vpn:5500/md_recorder:${IMAGE_VERSION} + container_name: ${CBCD_CONTAINER_NAME} + user: ${CVTT_USER:-1001:1001} + pull_policy: always + restart: unless-stopped + environment: + CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789} + CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg} + CREDS_FILE: ${CREDS_FILE} + EXCHANGE_ACCT_NAME: ${CBCD_EXCHANGE_ACCT_NAME} + INSTRUMENT_GROUP: ${CBCD_INSTRUMENT_GROUP} + DB_CRED_KEY: ${CBCD_DB_CRED_KEY} + LOG_FILE: ${CBCD_LOG_FILE} + ADD_ARGS: ${CBCD_ADD_ARGS:-} + volumes: + - ./config:/config + - ./logs:/logs + ports: + - ${CBCD_ADMIN_PORT}:7200 diff --git a/market_data/md_recorder/deployment/cvtt-md/md_recorder.cfg b/market_data/md_recorder/deployment/cvtt-md/md_recorder.cfg new file mode 100644 index 0000000..89c7ba6 --- /dev/null +++ b/market_data/md_recorder/deployment/cvtt-md/md_recorder.cfg @@ -0,0 +1,206 @@ +{ + "refdata": { + "assets": @inc=http://@env{CONFIG_SERVICE}/refdata/assets + , "instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/instruments + , "exchange_instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/exchange_instruments + , "dynamic_instrument_exchanges": ["ALPACA"] + , "exchanges": { + "COINBASE_AT": { + "exchange_id": "COINBASE" + , "name": "Coinbase Advanced Trade" + , "type": "SPOT" + + , "mktdata" : { + # ****** NEW VAlUE + # "class": "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMultiWsFeed" + "class": "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMdFeed" + # ****** NEW VAlUE + + , "reconnect_secs": 2 + , "max_connections": 10 + , "price_throttle": true + + , "ws_url": ["WSS", "wss://ws-feed.exchange.coinbase.com"] + } + } + , "BNBSPOT": { + "exchange_id": "BNBSPOT" + , "name": "Binance Spot" + , "type": "SPOT" + + , "mktdata" : { + # -------- NEW VALUE + # "class": "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMultiWsFeed" + "class": "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMktDataFeed" + # -------- NEW VALUE + , "max_connections": 10 + , "reconnect_secs": 2.0 + + + , "ws_url": ["WSS", "wss://stream.binance.com/ws"] + , "REST": { + "depth_snapshot_url": ["GET", "https://api.binance.com/api/v3/depth"], + } + } + } + , "BNBFUT": { + "exchange_id": "BNBFUT" + , "name": "Binance Futures USD-M" + , "type": "FUTURES" + + , "mktdata" : { + # -------- NEW VALUE + # "class": "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMultiWsFeed" + "class": "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMktDataFeed" + # -------- NEW VALUE + , "max_connections": 10 + , "reconnect_secs": 2.0 + + , "ws_url": ["WSS", "wss://fstream.binance.com/ws"] + + , "REST": { + # https://fapi.binance.com/fapi/v1/depth?symbol=BNBBTC&limit=1000 + "depth_snapshot_url": ["GET", "https://fapi.binance.com/fapi/v1/depth"], + } + } + } + } + } + , "comm": @inc=http://@env{CONFIG_SERVICE}/apps/common/comm + + , "md_recorder": { + "instrument_groups": { + "BNBFUT_CLD21": [ + "BNBFUT:PERP-BTC-USDT" + , "BNBFUT:PERP-ETH-USDT" + ] + "BNBFUT_CVTTDATA": [ + "BNBFUT:PERP-BTC-USDT" + , "BNBFUT:PERP-ETH-USDT" + ] + , "BNBSPOT_CLD21": [ + "BNBSPOT:PAIR-ADA-USDT" + , "BNBSPOT:PAIR-BCH-USDT" + , "BNBSPOT:PAIR-BTC-USDT" + , "BNBSPOT:PAIR-DOT-USDT" + , "BNBSPOT:PAIR-ETH-USDT" + , "BNBSPOT:PAIR-LTC-USDT" + , "BNBSPOT:PAIR-SOL-USDT" + , "BNBSPOT:PAIR-USDC-USDT" + , "BNBSPOT:PAIR-XRP-USDT" + ] + , "BNBSPOT_CVTTDATA": [ + "BNBSPOT:PAIR-ADA-USDT" + , "BNBSPOT:PAIR-BCH-USDT" + , "BNBSPOT:PAIR-BTC-USDT" + , "BNBSPOT:PAIR-DOT-USDT" + , "BNBSPOT:PAIR-ETH-USDT" + , "BNBSPOT:PAIR-LTC-USDT" + , "BNBSPOT:PAIR-SOL-USDT" + , "BNBSPOT:PAIR-USDC-USDT" + , "BNBSPOT:PAIR-XRP-USDT" + ] + , "COINBASE_CLD21": [ + "COINBASE:PAIR-ADA-USD" + , "COINBASE:PAIR-BCH-USD" + , "COINBASE:PAIR-BTC-USD" + , "COINBASE:PAIR-DOT-USD" + , "COINBASE:PAIR-ETH-USD" + , "COINBASE:PAIR-LTC-USD" + , "COINBASE:PAIR-SOL-USD" + , "COINBASE:PAIR-XRP-USD" + ] + , "COINBASE_CVTTDATA": [ + "COINBASE:PAIR-ADA-USD" + , "COINBASE:PAIR-BCH-USD" + , "COINBASE:PAIR-BTC-USD" + , "COINBASE:PAIR-DOT-USD" + , "COINBASE:PAIR-ETH-USD" + , "COINBASE:PAIR-LTC-USD" + , "COINBASE:PAIR-SOL-USD" + , "COINBASE:PAIR-XRP-USD" + ] + } + , "db": { + "class": "cvttpy.tools.db.timescale.tsdb_client.TimescaleDbClient" + } + , "flush_interval_sec": 0.5 + + , "collections": { + "__dummy__": null + , "ALPACA": { + # dynamic instruments names + "tables": { + "trades": "alpaca_iex.md_trades" + , "booktops": "alpaca_iex.md_booktops" + , "booksnaps": "alpaca_iex.md_booksnaps" + } + } + , "BNBSPOT": { + "instruments": { + "PAIR-ADA-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-BCH-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-BTC-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-DOT-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-ETH-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-LTC-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-SOL-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-USDC-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-XRP-USDT": ["TRADES", "BOOK_TOP"] + } + , "tables": { + "trades": "bnbspot.md_trades" + , "booktops": "bnbspot.md_booktops" + , "booksnaps": "bnbspot.md_booksnaps" + } + } + , "BNBFUT": { + "instruments": { + "PERP-BTC-USDT": ["TRADES", "BOOK_TOP"] + , "PERP-ETH-USDT": ["TRADES", "BOOK_TOP"] + } + , "tables": { + "trades": "bnbfut.md_trades" + , "booktops": "bnbfut.md_booktops" + , "booksnaps": "bnbfut.md_booksnaps" + } + } + , "COINBASE": { + "instruments": { + "PAIR-ADA-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-BCH-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-BTC-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-DOT-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-ETH-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-LTC-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-SOL-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-XRP-USD": ["TRADES", "BOOK_TOP"] + } + , "tables": { + "trades": "coinbase.md_trades" + , "booktops": "coinbase.md_booktops" + , "booksnaps": "coinbase.md_booksnaps" + } + } + } + } + + # ---------------------------------- + , "exchanges_settings": { + "*": { + "order_entry": { + "enabled": false + , "streaming": false + }, + "mkt_data": { + "enabled": true + , "streaming": true + }, + "accounting": { + "enabled": false + , "streaming": false + } + } + } +} + diff --git a/market_data/md_recorder/entrypoint.sh b/market_data/md_recorder/entrypoint.sh new file mode 100755 index 0000000..a40916f --- /dev/null +++ b/market_data/md_recorder/entrypoint.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# runs in container + + +Cmd="python3.12" +Cmd+=" cvttpy/apps/md/md_recorder.py" + +Cmd+=" --config=${CONFIG_FILE:-/config/cvtt_musvc.cfg}" +Cmd+=" --credentials_file=${CREDS_FILE:-/config/.creds}" + +Cmd="${Cmd} --instrument_group=${INSTRUMENT_GROUP}" +Cmd="${Cmd} --exch_acct_cfgname=${EXCHANGE_ACCT_NAME}" +Cmd="${Cmd} --db_credentials_key=${DB_CRED_KEY}" +Cmd="${Cmd} --admin_port=7200" + +Cmd+=" --log_file=${LOG_FILE:-/logs/%T.MD_REC.${INSTRUMENT_GROUP}.log}" +Cmd+=" ${ADD_ARGS}" + +echo ${Cmd} +exec ${Cmd} + diff --git a/market_data/md_recorder/requirements.txt b/market_data/md_recorder/requirements.txt index 2c7a918..6edb609 100644 --- a/market_data/md_recorder/requirements.txt +++ b/market_data/md_recorder/requirements.txt @@ -5,3 +5,4 @@ hjson>=3.1.0 pandas>=1.5.3 sortedcontainers>=2.4.0 redis>=5.0.8 +debugpy>=1.8.13 diff --git a/market_data/md_recorder/run_md_recorder.sh b/market_data/md_recorder/run_md_recorder.sh deleted file mode 100755 index 4b1a8f4..0000000 --- a/market_data/md_recorder/run_md_recorder.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -# runs in container - -ValidJobs=('BNBFUT_CLD21' 'BNBSPOT_CLD21' 'COINBASE_CLD21' 'BNBFUT_CVTTDATA' 'BNBSPOT_CVTTDATA' 'COINBASE_CVTTDATA') -# runs on host to start container -usage() { - echo "Usage: $0 " - exit 1 -} - -is_valid() { - local job=$1 - for valid_job in "${ValidJobs[@]}"; - do - # echo "job=$job valid_job=$valid_job" - if [[ "${job}" == "${valid_job}" ]]; then - return 0 - fi - done - return 1 -} - -job=${1} -if ! is_valid "${job}"; then - usage -fi - -ConfigServer=cloud16.cvtt.vpn:6789 - -Cmd="python3.12" -Cmd="${Cmd} cvttpy/apps/md/md_recorder.py" -Cmd="${Cmd} --config=http://${ConfigServer}/apps/md_recorder" -Cmd="${Cmd} --credentials_file=/.creds" -Cmd="${Cmd} --compress_log" -Cmd="${Cmd} --instrument_group=${job}" -Cmd="${Cmd} --log_file=/logs/%T.MD_REC.${job}.log" -if [ "${job}" == "BNBFUT_CLD21" ] ; then - Cmd="${Cmd} --exch_acct_cfgname=BNBFUT" - Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21" - Cmd="${Cmd} --admin_port=7201" -elif [ "${job}" == "COINBASE_CLD21" ] ; then - Cmd="${Cmd} --exch_acct_cfgname=COINBASE_AT" - Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21" - Cmd="${Cmd} --admin_port=7202" -elif [ "${job}" == "BNBSPOT_CLD21" ] ; then - Cmd="${Cmd} --exch_acct_cfgname=BNBSPOT" - Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21" - Cmd="${Cmd} --admin_port=7203" -elif [ "${job}" == "BNBSPOT_CVTTDATA" ] ; then - Cmd="${Cmd} --exch_acct_cfgname=BNBSPOT" - Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA" - Cmd="${Cmd} --admin_port=7204" -elif [ "${job}" == "BNBFUT_CVTTDATA" ] ; then - Cmd="${Cmd} --exch_acct_cfgname=BNBFUT" - Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA" - Cmd="${Cmd} --admin_port=7205" -elif [ "${job}" == "COINBASE_CVTTDATA" ] ; then - Cmd="${Cmd} --exch_acct_cfgname=COINBASE_AT" - Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA" - Cmd="${Cmd} --admin_port=7206" -else - echo "Unrecognized JOB: ${job}" - exit 1 -fi -echo ${Cmd} -eval ${Cmd} - - diff --git a/microservices/mu_svc_functions.sh b/microservices/mu_svc_functions.sh index c9e3351..be3e65a 100755 --- a/microservices/mu_svc_functions.sh +++ b/microservices/mu_svc_functions.sh @@ -17,7 +17,7 @@ get_project_version() { echo "Checking ${user}@${host} for ${rel_dir}/${Project}/${Version} ..." if ssh -q -p ${port} ${user}@${host} "test -d ${rel_dir}/${Project}/${Version}" then - echo "Version ${Version} found..." + echo "Getting Version \"${Version}\"..." rsync_cmd="rsync -ahv -e \"ssh -p ${port}\"" rsync_cmd="${rsync_cmd} ${user}@${host}:${rel_dir}/${Project}/${Version}/" rsync_cmd="${rsync_cmd} ./" diff --git a/mu_svc_functions.sh b/mu_svc_functions.sh new file mode 100755 index 0000000..be3e65a --- /dev/null +++ b/mu_svc_functions.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +get_project_version() { + Project=${1} + Version=${2} + + host="cloud21.cvtt.vpn" + port="22" + user="cvttdist" + rel_dir="/home/cvttdist/software/cvtt2" + + if [ "${Version}" == "latest" ]; then + echo "Checking for latest version of ${Project} on ${user}@${host}:${rel_dir}" + Version=$(ssh -q -p ${port} ${user}@${host} "ls -tr ${rel_dir}/${Project} | tail -1" ) + echo "Latest version is ${Version}" + fi + echo "Checking ${user}@${host} for ${rel_dir}/${Project}/${Version} ..." + if ssh -q -p ${port} ${user}@${host} "test -d ${rel_dir}/${Project}/${Version}" + then + echo "Getting Version \"${Version}\"..." + rsync_cmd="rsync -ahv -e \"ssh -p ${port}\"" + rsync_cmd="${rsync_cmd} ${user}@${host}:${rel_dir}/${Project}/${Version}/" + rsync_cmd="${rsync_cmd} ./" + echo ${rsync_cmd} + eval ${rsync_cmd} + status=$? + if [ ${status} -eq 0 ] + then + echo "Loading successful..." + return 0 + else + return 1 + fi + else + echo "Not Found ${rel_dir}/${Project}/${Version} on ${user}@${host}" + return 1 + fi +} + +build_docker_image() { + ImageName=${1} + RegistryService=${2} + Version=${3} + Project=${4} + ProjectVersion=${5} + + Cmd="docker build" + if [ "${ProjectVersion}" != "" ]; then + Cmd+=" --label ${Project}=\"${ProjectVersion}\"" + fi + Cmd+=" -t ${ImageName}" + Cmd+=" -t ${ImageName}:latest" + Cmd+=" -t ${ImageName}:${Version}" + Cmd+=" -f Dockerfile" + Cmd+=" ." + echo ${Cmd} + eval ${Cmd} || exit + + Cmd="docker tag ${ImageName}:latest ${RegistryService}/${ImageName}:latest" + echo ${Cmd} + eval ${Cmd} || exit + + Cmd="docker tag ${ImageName}:${Version} ${RegistryService}/${ImageName}:${Version}" + echo ${Cmd} + eval ${Cmd} || exit + + Cmd="docker push ${RegistryService}/${ImageName}:latest" + echo ${Cmd} + eval ${Cmd} || exit + + Cmd="docker push ${RegistryService}/${ImageName}:${Version}" + echo ${Cmd} + eval ${Cmd} || exit +} + +