commit 7fba7f9406e512f99829d5c13da381fc89dadd65 Author: Cryptoval Trading Technologies Date: Sat May 17 01:45:34 2025 +0000 initial diff --git a/.env b/.env new file mode 100644 index 0000000..12140cf --- /dev/null +++ b/.env @@ -0,0 +1,53 @@ +CVTT_USER=1001:1001 +MD_RECORDER_VERSION=0.1.1 + +CONFIG_FILE=/config/md_recorder.cfg +CREDS_FILE=/config/.creds + +# coinbase-cvttdata +CB24_DEBUG=N +CB24_DEBUG_WAIT=Y +CB24_DEBUG_PORT=45678 + +CB24_CONTAINER_NAME=md_recorder.COINBASE_CLOUD24 +CB24_INSTRUMENT_GROUP=COINBASE_CLOUD24 +CB24_EXCHANGE_ACCT_NAME=COINBASE_AT +CB24_DB_CRED_KEY=TSDB_MD_CLOUD24 + +CB24_LOG_FILE=/logs/%T.MD_REC.${CB24_INSTRUMENT_GROUP}.log +CB24_ADD_ARGS=--compress_log + +CB24_ADMIN_PORT=8201 + + +# bnspot-cvttdata +BS24_DEBUG=N +BS24_DEBUG_WAIT=Y +BS24_DEBUG_PORT=45679 + +BS24_CONTAINER_NAME=md_recorder.BNBSPOT_CLOUD24 +BS24_INSTRUMENT_GROUP=BNBSPOT_CLOUD24 +BS24_EXCHANGE_ACCT_NAME=BNBSPOT +BS24_DB_CRED_KEY=TSDB_MD_CLOUD24 + +BS24_LOG_FILE=/logs/%T.MD_REC.${BS24_INSTRUMENT_GROUP}.log +BS24_ADD_ARGS=--compress_log + +BS24_ADMIN_PORT=8202 + +# bnbfut-cvttdata +BF24_DEBUG=N +BF24_DEBUG_WAIT=Y +BF24_DEBUG_PORT=45680 + +BF24_CONTAINER_NAME=md_recorder.BNBFUT_CLOUD24 +BF24_INSTRUMENT_GROUP=BNBFUT_CLOUD24 +BF24_EXCHANGE_ACCT_NAME=BNBFUT +BF24_DB_CRED_KEY=TSDB_MD_CLOUD24 + +BF24_LOG_FILE=/logs/%T.MD_REC.${BF24_INSTRUMENT_GROUP}.log +BF24_ADD_ARGS=--compress_log + +BF24_ADMIN_PORT=8203 + + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5eb6531 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +data/ +logs/ diff --git a/config/.creds b/config/.creds new file mode 100644 index 0000000..6bef24e --- /dev/null +++ b/config/.creds @@ -0,0 +1,24 @@ +{ + "__dummy__": null + , "TSDB_MD_CLD21": { + "host": "cloud21.cvtt.vpn" + , "port": 5432 + , "user": "cvtt" + , "database": "cvtt_md" + , "password": "ICdIh0JnMM7vM7Pf" + }, + "TSDB_MD_CLOUD28": { + "host": "cloud28.cvtt.vpn" + , "port": 55432 + , "user": "cvtt" + , "database": "cvtt_md" + , "password": "320ea18a54b5bed71371256bd9b68d31" + }, + "TSDB_MD_CLOUD24": { + "host": "cloud24.cvtt.vpn" + , "port": 55432 + , "user": "cvtt" + , "database": "cvtt_md" + , "password": "23a8c597a800d44906d01483e8940e77" + } +} diff --git a/config/md_recorder.cfg b/config/md_recorder.cfg new file mode 100644 index 0000000..bd133de --- /dev/null +++ b/config/md_recorder.cfg @@ -0,0 +1,207 @@ +{ + "refdata": { + "assets": @inc=http://@env{CONFIG_SERVICE}/refdata/assets + , "instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/instruments + , "exchange_instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/exchange_instruments + , "dynamic_instrument_exchanges": ["ALPACA"] + , "exchanges": { + "COINBASE_AT": { + "exchange_id": "COINBASE" + , "name": "Coinbase Advanced Trade" + , "type": "SPOT" + + , "mktdata" : { + "class": "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMdFeed" + + , "reconnect_secs": 2 + , "max_connections": 10 + , "price_throttle": true + + , "api": { + "web_socket": { + "url": "wss://ws-feed.exchange.coinbase.com" + } + } + } + } + , "BNBSPOT": { + "exchange_id": "BNBSPOT" + , "name": "Binance Spot" + , "type": "SPOT" + + , "mktdata" : { + "class": "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMktDataFeed" + , "max_connections": 10 + , "reconnect_secs": 2.0 + + + , "api": { + "web_socket": { + "url": "wss://stream.binance.com/ws" + }, + "REST": { + "depth_snapshot_url": ["GET", "https://api.binance.com/api/v3/depth"], + } + } + } + } + , "BNBFUT": { + "exchange_id": "BNBFUT" + , "name": "Binance Futures USD-M" + , "type": "FUTURES" + + , "mktdata" : { + "class": "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMktDataFeed" + , "max_connections": 10 + , "reconnect_secs": 2.0 + + , "api": { + "web_socket": { + "url": "wss://fstream.binance.com/ws" + }, + "REST": { + "depth_snapshot_url": ["GET", "https://fapi.binance.com/fapi/v1/depth"], + } + } + } + } + } + } + , "comm": @inc=http://@env{CONFIG_SERVICE}/apps/common/comm + + , "md_recorder": { + "instrument_groups": { + "BNBFUT_CLOUD24": [ + "BNBFUT:PERP-BTC-USDT" + , "BNBFUT:PERP-ETH-USDT" + ], + "BNBFUT_CLOUD28": [ + "BNBFUT:PERP-BTC-USDT" + , "BNBFUT:PERP-ETH-USDT" + ], + "BNBSPOT_CLOUD24": [ + "BNBSPOT:PAIR-ADA-USDT" + , "BNBSPOT:PAIR-BCH-USDT" + , "BNBSPOT:PAIR-BTC-USDT" + , "BNBSPOT:PAIR-DOT-USDT" + , "BNBSPOT:PAIR-ETH-USDT" + , "BNBSPOT:PAIR-LTC-USDT" + , "BNBSPOT:PAIR-SOL-USDT" + , "BNBSPOT:PAIR-USDC-USDT" + , "BNBSPOT:PAIR-XRP-USDT" + ], + "BNBSPOT_CLOUD28": [ + "BNBSPOT:PAIR-ADA-USDT" + , "BNBSPOT:PAIR-BCH-USDT" + , "BNBSPOT:PAIR-BTC-USDT" + , "BNBSPOT:PAIR-DOT-USDT" + , "BNBSPOT:PAIR-ETH-USDT" + , "BNBSPOT:PAIR-LTC-USDT" + , "BNBSPOT:PAIR-SOL-USDT" + , "BNBSPOT:PAIR-USDC-USDT" + , "BNBSPOT:PAIR-XRP-USDT" + ], + "COINBASE_CLOUD24": [ + "COINBASE:PAIR-ADA-USD" + , "COINBASE:PAIR-BCH-USD" + , "COINBASE:PAIR-BTC-USD" + , "COINBASE:PAIR-DOT-USD" + , "COINBASE:PAIR-ETH-USD" + , "COINBASE:PAIR-LTC-USD" + , "COINBASE:PAIR-SOL-USD" + , "COINBASE:PAIR-XRP-USD" + ], + "COINBASE_CLOUD28": [ + "COINBASE:PAIR-ADA-USD" + , "COINBASE:PAIR-BCH-USD" + , "COINBASE:PAIR-BTC-USD" + , "COINBASE:PAIR-DOT-USD" + , "COINBASE:PAIR-ETH-USD" + , "COINBASE:PAIR-LTC-USD" + , "COINBASE:PAIR-SOL-USD" + , "COINBASE:PAIR-XRP-USD" + ] + }, + "db": { + "class": "cvttpy.tools.db.timescale.tsdb_client.TimescaleDbClient" + }, + "flush_interval_sec": 0.5, + + "collections": { + "__dummy__": null + , "ALPACA": { + # dynamic instruments names + "tables": { + "trades": "alpaca_iex.md_trades" + , "booktops": "alpaca_iex.md_booktops" + , "booksnaps": "alpaca_iex.md_booksnaps" + } + } + , "BNBSPOT": { + "instruments": { + "PAIR-ADA-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-BCH-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-BTC-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-DOT-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-ETH-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-LTC-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-SOL-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-USDC-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-XRP-USDT": ["TRADES", "BOOK_TOP"] + } + , "tables": { + "trades": "bnbspot.md_trades" + , "booktops": "bnbspot.md_booktops" + , "booksnaps": "bnbspot.md_booksnaps" + } + } + , "BNBFUT": { + "instruments": { + "PERP-BTC-USDT": ["TRADES", "BOOK_TOP"] + , "PERP-ETH-USDT": ["TRADES", "BOOK_TOP"] + } + , "tables": { + "trades": "bnbfut.md_trades" + , "booktops": "bnbfut.md_booktops" + , "booksnaps": "bnbfut.md_booksnaps" + } + } + , "COINBASE": { + "instruments": { + "PAIR-ADA-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-BCH-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-BTC-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-DOT-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-ETH-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-LTC-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-SOL-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-XRP-USD": ["TRADES", "BOOK_TOP"] + } + , "tables": { + "trades": "coinbase.md_trades" + , "booktops": "coinbase.md_booktops" + , "booksnaps": "coinbase.md_booksnaps" + } + } + } + } + + # ---------------------------------- + "exchanges_settings": { + "*": { + "order_entry": { + "enabled": false + , "streaming": false + }, + "mkt_data": { + "enabled": true + , "streaming": true + }, + "accounting": { + "enabled": false + , "streaming": false + } + } + } +} + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..278a2e8 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,95 @@ +services: + + cbspot24_md_recorder: + image: cloud21.cvtt.vpn:5500/md_recorder:${MD_RECORDER_VERSION} + container_name: ${CB24_CONTAINER_NAME} + user: ${CVTT_USER:-1001:1001} + pull_policy: always + restart: unless-stopped + environment: + CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789} + CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg} + CREDS_FILE: ${CREDS_FILE} + EXCHANGE_ACCT_NAME: ${CB24_EXCHANGE_ACCT_NAME} + INSTRUMENT_GROUP: ${CB24_INSTRUMENT_GROUP} + DB_CRED_KEY: ${CB24_DB_CRED_KEY} + LOG_FILE: ${CB24_LOG_FILE} + ADD_ARGS: ${CB24_ADD_ARGS:-} + MDREC_DEBUG: ${CB24_DEBUG:-N} + MDREC_DEBUG_WAIT: ${CB24_DEBUG_WAIT:-N} + volumes: + - ./config:/config + - ./logs:/logs + ports: + - ${CB24_ADMIN_PORT}:7200 + - ${CB24_DEBUG_PORT}:5678 + depends_on: + - timescaledb + + bnbspot24_md_recorder: + image: cloud21.cvtt.vpn:5500/md_recorder:${MD_RECORDER_VERSION} + container_name: ${BS24_CONTAINER_NAME} + user: ${CVTT_USER:-1001:1001} + pull_policy: always + restart: unless-stopped + environment: + CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789} + CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg} + CREDS_FILE: ${CREDS_FILE} + EXCHANGE_ACCT_NAME: ${BS24_EXCHANGE_ACCT_NAME} + INSTRUMENT_GROUP: ${BS24_INSTRUMENT_GROUP} + DB_CRED_KEY: ${BS24_DB_CRED_KEY} + LOG_FILE: ${BS24_LOG_FILE} + ADD_ARGS: ${BS24_ADD_ARGS:-} + MDREC_DEBUG: ${BS24_DEBUG:-N} + MDREC_DEBUG_WAIT: ${BS24_DEBUG_WAIT:-N} + volumes: + - ./config:/config + - ./logs:/logs + ports: + - ${BS24_ADMIN_PORT}:7200 + - ${BS24_DEBUG_PORT}:5678 + depends_on: + - timescaledb + +# bnbfut24_md_recorder: +# image: cloud21.cvtt.vpn:5500/md_recorder:${MD_RECORDER_VERSION} +# container_name: ${BF24_CONTAINER_NAME} +# user: ${CVTT_USER:-1001:1001} +# pull_policy: always +# restart: unless-stopped +# environment: +# CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789} +# CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg} +# CREDS_FILE: ${CREDS_FILE} +# EXCHANGE_ACCT_NAME: ${BF24_EXCHANGE_ACCT_NAME} +# INSTRUMENT_GROUP: ${BF24_INSTRUMENT_GROUP} +# DB_CRED_KEY: ${BF24_DB_CRED_KEY} +# LOG_FILE: ${BF24_LOG_FILE} +# ADD_ARGS: ${BF24_ADD_ARGS:-} +# MDREC_DEBUG: ${BF24_DEBUG:-N} +# MDREC_DEBUG_WAIT: ${BF24_DEBUG_WAIT:-N} +# volumes: +# - ./config:/config +# - ./logs:/logs +# ports: +# - ${BF24_ADMIN_PORT}:7200 +# - ${BF24_DEBUG_PORT}:5678 +# depends_on: +# - timescaledb + + timescaledb: + image: cloud21.cvtt.vpn:5500/md_recorder-tsdb:${MD_RECORDER_VERSION} + container_name: md_recorder-tsdb + pull_policy: always + environment: + - POSTGRES_PASSWORD=postgres + - POSTGRES_USER=postgres + - POSTGRES_DB=postgres + ports: + - "55432:5432" + volumes: + - ./data/timescaledb_data:/var/lib/postgresql/data + restart: unless-stopped + +