diff --git a/config_service/.dockerignore b/__DECO__/config_service/.dockerignore similarity index 100% rename from config_service/.dockerignore rename to __DECO__/config_service/.dockerignore diff --git a/config_service/Dockerfile b/__DECO__/config_service/Dockerfile similarity index 100% rename from config_service/Dockerfile rename to __DECO__/config_service/Dockerfile diff --git a/config_service/HOWTO.md b/__DECO__/config_service/HOWTO.md similarity index 100% rename from config_service/HOWTO.md rename to __DECO__/config_service/HOWTO.md diff --git a/config_service/requirements.txt b/__DECO__/config_service/requirements.txt similarity index 100% rename from config_service/requirements.txt rename to __DECO__/config_service/requirements.txt diff --git a/market_data/md_recorder/.dockerignore b/__DECO__/md_recorder/.dockerignore similarity index 100% rename from market_data/md_recorder/.dockerignore rename to __DECO__/md_recorder/.dockerignore diff --git a/market_data/md_recorder/Dockerfile b/__DECO__/md_recorder/Dockerfile similarity index 100% rename from market_data/md_recorder/Dockerfile rename to __DECO__/md_recorder/Dockerfile diff --git a/__DECO__/md_recorder/VERSION b/__DECO__/md_recorder/VERSION new file mode 100644 index 0000000..99d85ec --- /dev/null +++ b/__DECO__/md_recorder/VERSION @@ -0,0 +1 @@ +0.0.6 \ No newline at end of file diff --git a/market_data/md_recorder/build.sh b/__DECO__/md_recorder/build.sh similarity index 90% rename from market_data/md_recorder/build.sh rename to __DECO__/md_recorder/build.sh index d01c6cb..5ea8af4 100755 --- a/market_data/md_recorder/build.sh +++ b/__DECO__/md_recorder/build.sh @@ -4,10 +4,11 @@ RegistryService=cloud21.cvtt.vpn:5500 ImageName=md_recorder BashFunctionCode=$(realpath $(dirname ${0}))/../../dckr_bld_functions.sh +ImageVersion=$(cat $(realpath $(dirname ${0}))/VERSION) # --- Settings usage() { - echo "Usage: $0 -V -L :" + echo "Usage: $0 -L : [-V ] " exit 1 } diff --git a/__DECO__/md_recorder/deployment/cloud24/.creds b/__DECO__/md_recorder/deployment/cloud24/.creds new file mode 100644 index 0000000..1301a78 --- /dev/null +++ b/__DECO__/md_recorder/deployment/cloud24/.creds @@ -0,0 +1,17 @@ +{ + "__dummy__": null + , "TSDB_MD_CLD21": { + "host": "cloud21.cvtt.vpn" + , "port": 5432 + , "user": "cvtt" + , "database": "cvtt_md" + , "password": "ICdIh0JnMM7vM7Pf" + } + , "TSDB_MD_CVTTDATA": { + "host": "cvttdata.cvtt.vpn" + , "port": 5432 + , "user": "cvtt" + , "database": "cvtt_md" + , "password": "ICdIh0JnMM7vM7Pf" + } +} diff --git a/market_data/md_recorder/deployment/cloud24/.env b/__DECO__/md_recorder/deployment/cloud24/.env similarity index 100% rename from market_data/md_recorder/deployment/cloud24/.env rename to __DECO__/md_recorder/deployment/cloud24/.env diff --git a/market_data/md_recorder/deployment/cloud24/docker-compose.yml b/__DECO__/md_recorder/deployment/cloud24/docker-compose.yml similarity index 100% rename from market_data/md_recorder/deployment/cloud24/docker-compose.yml rename to __DECO__/md_recorder/deployment/cloud24/docker-compose.yml diff --git a/__DECO__/md_recorder/deployment/cloud24/md_recorder.cfg b/__DECO__/md_recorder/deployment/cloud24/md_recorder.cfg new file mode 100644 index 0000000..89c7ba6 --- /dev/null +++ b/__DECO__/md_recorder/deployment/cloud24/md_recorder.cfg @@ -0,0 +1,206 @@ +{ + "refdata": { + "assets": @inc=http://@env{CONFIG_SERVICE}/refdata/assets + , "instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/instruments + , "exchange_instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/exchange_instruments + , "dynamic_instrument_exchanges": ["ALPACA"] + , "exchanges": { + "COINBASE_AT": { + "exchange_id": "COINBASE" + , "name": "Coinbase Advanced Trade" + , "type": "SPOT" + + , "mktdata" : { + # ****** NEW VAlUE + # "class": "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMultiWsFeed" + "class": "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMdFeed" + # ****** NEW VAlUE + + , "reconnect_secs": 2 + , "max_connections": 10 + , "price_throttle": true + + , "ws_url": ["WSS", "wss://ws-feed.exchange.coinbase.com"] + } + } + , "BNBSPOT": { + "exchange_id": "BNBSPOT" + , "name": "Binance Spot" + , "type": "SPOT" + + , "mktdata" : { + # -------- NEW VALUE + # "class": "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMultiWsFeed" + "class": "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMktDataFeed" + # -------- NEW VALUE + , "max_connections": 10 + , "reconnect_secs": 2.0 + + + , "ws_url": ["WSS", "wss://stream.binance.com/ws"] + , "REST": { + "depth_snapshot_url": ["GET", "https://api.binance.com/api/v3/depth"], + } + } + } + , "BNBFUT": { + "exchange_id": "BNBFUT" + , "name": "Binance Futures USD-M" + , "type": "FUTURES" + + , "mktdata" : { + # -------- NEW VALUE + # "class": "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMultiWsFeed" + "class": "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMktDataFeed" + # -------- NEW VALUE + , "max_connections": 10 + , "reconnect_secs": 2.0 + + , "ws_url": ["WSS", "wss://fstream.binance.com/ws"] + + , "REST": { + # https://fapi.binance.com/fapi/v1/depth?symbol=BNBBTC&limit=1000 + "depth_snapshot_url": ["GET", "https://fapi.binance.com/fapi/v1/depth"], + } + } + } + } + } + , "comm": @inc=http://@env{CONFIG_SERVICE}/apps/common/comm + + , "md_recorder": { + "instrument_groups": { + "BNBFUT_CLD21": [ + "BNBFUT:PERP-BTC-USDT" + , "BNBFUT:PERP-ETH-USDT" + ] + "BNBFUT_CVTTDATA": [ + "BNBFUT:PERP-BTC-USDT" + , "BNBFUT:PERP-ETH-USDT" + ] + , "BNBSPOT_CLD21": [ + "BNBSPOT:PAIR-ADA-USDT" + , "BNBSPOT:PAIR-BCH-USDT" + , "BNBSPOT:PAIR-BTC-USDT" + , "BNBSPOT:PAIR-DOT-USDT" + , "BNBSPOT:PAIR-ETH-USDT" + , "BNBSPOT:PAIR-LTC-USDT" + , "BNBSPOT:PAIR-SOL-USDT" + , "BNBSPOT:PAIR-USDC-USDT" + , "BNBSPOT:PAIR-XRP-USDT" + ] + , "BNBSPOT_CVTTDATA": [ + "BNBSPOT:PAIR-ADA-USDT" + , "BNBSPOT:PAIR-BCH-USDT" + , "BNBSPOT:PAIR-BTC-USDT" + , "BNBSPOT:PAIR-DOT-USDT" + , "BNBSPOT:PAIR-ETH-USDT" + , "BNBSPOT:PAIR-LTC-USDT" + , "BNBSPOT:PAIR-SOL-USDT" + , "BNBSPOT:PAIR-USDC-USDT" + , "BNBSPOT:PAIR-XRP-USDT" + ] + , "COINBASE_CLD21": [ + "COINBASE:PAIR-ADA-USD" + , "COINBASE:PAIR-BCH-USD" + , "COINBASE:PAIR-BTC-USD" + , "COINBASE:PAIR-DOT-USD" + , "COINBASE:PAIR-ETH-USD" + , "COINBASE:PAIR-LTC-USD" + , "COINBASE:PAIR-SOL-USD" + , "COINBASE:PAIR-XRP-USD" + ] + , "COINBASE_CVTTDATA": [ + "COINBASE:PAIR-ADA-USD" + , "COINBASE:PAIR-BCH-USD" + , "COINBASE:PAIR-BTC-USD" + , "COINBASE:PAIR-DOT-USD" + , "COINBASE:PAIR-ETH-USD" + , "COINBASE:PAIR-LTC-USD" + , "COINBASE:PAIR-SOL-USD" + , "COINBASE:PAIR-XRP-USD" + ] + } + , "db": { + "class": "cvttpy.tools.db.timescale.tsdb_client.TimescaleDbClient" + } + , "flush_interval_sec": 0.5 + + , "collections": { + "__dummy__": null + , "ALPACA": { + # dynamic instruments names + "tables": { + "trades": "alpaca_iex.md_trades" + , "booktops": "alpaca_iex.md_booktops" + , "booksnaps": "alpaca_iex.md_booksnaps" + } + } + , "BNBSPOT": { + "instruments": { + "PAIR-ADA-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-BCH-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-BTC-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-DOT-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-ETH-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-LTC-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-SOL-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-USDC-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-XRP-USDT": ["TRADES", "BOOK_TOP"] + } + , "tables": { + "trades": "bnbspot.md_trades" + , "booktops": "bnbspot.md_booktops" + , "booksnaps": "bnbspot.md_booksnaps" + } + } + , "BNBFUT": { + "instruments": { + "PERP-BTC-USDT": ["TRADES", "BOOK_TOP"] + , "PERP-ETH-USDT": ["TRADES", "BOOK_TOP"] + } + , "tables": { + "trades": "bnbfut.md_trades" + , "booktops": "bnbfut.md_booktops" + , "booksnaps": "bnbfut.md_booksnaps" + } + } + , "COINBASE": { + "instruments": { + "PAIR-ADA-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-BCH-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-BTC-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-DOT-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-ETH-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-LTC-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-SOL-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-XRP-USD": ["TRADES", "BOOK_TOP"] + } + , "tables": { + "trades": "coinbase.md_trades" + , "booktops": "coinbase.md_booktops" + , "booksnaps": "coinbase.md_booksnaps" + } + } + } + } + + # ---------------------------------- + , "exchanges_settings": { + "*": { + "order_entry": { + "enabled": false + , "streaming": false + }, + "mkt_data": { + "enabled": true + , "streaming": true + }, + "accounting": { + "enabled": false + , "streaming": false + } + } + } +} + diff --git a/__DECO__/md_recorder/deployment/cvtt-md/.creds b/__DECO__/md_recorder/deployment/cvtt-md/.creds new file mode 100644 index 0000000..1301a78 --- /dev/null +++ b/__DECO__/md_recorder/deployment/cvtt-md/.creds @@ -0,0 +1,17 @@ +{ + "__dummy__": null + , "TSDB_MD_CLD21": { + "host": "cloud21.cvtt.vpn" + , "port": 5432 + , "user": "cvtt" + , "database": "cvtt_md" + , "password": "ICdIh0JnMM7vM7Pf" + } + , "TSDB_MD_CVTTDATA": { + "host": "cvttdata.cvtt.vpn" + , "port": 5432 + , "user": "cvtt" + , "database": "cvtt_md" + , "password": "ICdIh0JnMM7vM7Pf" + } +} diff --git a/market_data/md_recorder/deployment/cvtt-md/.env b/__DECO__/md_recorder/deployment/cvtt-md/.env similarity index 100% rename from market_data/md_recorder/deployment/cvtt-md/.env rename to __DECO__/md_recorder/deployment/cvtt-md/.env diff --git a/market_data/md_recorder/deployment/cvtt-md/docker-compose.yml b/__DECO__/md_recorder/deployment/cvtt-md/docker-compose.yml similarity index 100% rename from market_data/md_recorder/deployment/cvtt-md/docker-compose.yml rename to __DECO__/md_recorder/deployment/cvtt-md/docker-compose.yml diff --git a/__DECO__/md_recorder/deployment/cvtt-md/md_recorder.cfg b/__DECO__/md_recorder/deployment/cvtt-md/md_recorder.cfg new file mode 100644 index 0000000..89c7ba6 --- /dev/null +++ b/__DECO__/md_recorder/deployment/cvtt-md/md_recorder.cfg @@ -0,0 +1,206 @@ +{ + "refdata": { + "assets": @inc=http://@env{CONFIG_SERVICE}/refdata/assets + , "instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/instruments + , "exchange_instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/exchange_instruments + , "dynamic_instrument_exchanges": ["ALPACA"] + , "exchanges": { + "COINBASE_AT": { + "exchange_id": "COINBASE" + , "name": "Coinbase Advanced Trade" + , "type": "SPOT" + + , "mktdata" : { + # ****** NEW VAlUE + # "class": "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMultiWsFeed" + "class": "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMdFeed" + # ****** NEW VAlUE + + , "reconnect_secs": 2 + , "max_connections": 10 + , "price_throttle": true + + , "ws_url": ["WSS", "wss://ws-feed.exchange.coinbase.com"] + } + } + , "BNBSPOT": { + "exchange_id": "BNBSPOT" + , "name": "Binance Spot" + , "type": "SPOT" + + , "mktdata" : { + # -------- NEW VALUE + # "class": "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMultiWsFeed" + "class": "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMktDataFeed" + # -------- NEW VALUE + , "max_connections": 10 + , "reconnect_secs": 2.0 + + + , "ws_url": ["WSS", "wss://stream.binance.com/ws"] + , "REST": { + "depth_snapshot_url": ["GET", "https://api.binance.com/api/v3/depth"], + } + } + } + , "BNBFUT": { + "exchange_id": "BNBFUT" + , "name": "Binance Futures USD-M" + , "type": "FUTURES" + + , "mktdata" : { + # -------- NEW VALUE + # "class": "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMultiWsFeed" + "class": "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMktDataFeed" + # -------- NEW VALUE + , "max_connections": 10 + , "reconnect_secs": 2.0 + + , "ws_url": ["WSS", "wss://fstream.binance.com/ws"] + + , "REST": { + # https://fapi.binance.com/fapi/v1/depth?symbol=BNBBTC&limit=1000 + "depth_snapshot_url": ["GET", "https://fapi.binance.com/fapi/v1/depth"], + } + } + } + } + } + , "comm": @inc=http://@env{CONFIG_SERVICE}/apps/common/comm + + , "md_recorder": { + "instrument_groups": { + "BNBFUT_CLD21": [ + "BNBFUT:PERP-BTC-USDT" + , "BNBFUT:PERP-ETH-USDT" + ] + "BNBFUT_CVTTDATA": [ + "BNBFUT:PERP-BTC-USDT" + , "BNBFUT:PERP-ETH-USDT" + ] + , "BNBSPOT_CLD21": [ + "BNBSPOT:PAIR-ADA-USDT" + , "BNBSPOT:PAIR-BCH-USDT" + , "BNBSPOT:PAIR-BTC-USDT" + , "BNBSPOT:PAIR-DOT-USDT" + , "BNBSPOT:PAIR-ETH-USDT" + , "BNBSPOT:PAIR-LTC-USDT" + , "BNBSPOT:PAIR-SOL-USDT" + , "BNBSPOT:PAIR-USDC-USDT" + , "BNBSPOT:PAIR-XRP-USDT" + ] + , "BNBSPOT_CVTTDATA": [ + "BNBSPOT:PAIR-ADA-USDT" + , "BNBSPOT:PAIR-BCH-USDT" + , "BNBSPOT:PAIR-BTC-USDT" + , "BNBSPOT:PAIR-DOT-USDT" + , "BNBSPOT:PAIR-ETH-USDT" + , "BNBSPOT:PAIR-LTC-USDT" + , "BNBSPOT:PAIR-SOL-USDT" + , "BNBSPOT:PAIR-USDC-USDT" + , "BNBSPOT:PAIR-XRP-USDT" + ] + , "COINBASE_CLD21": [ + "COINBASE:PAIR-ADA-USD" + , "COINBASE:PAIR-BCH-USD" + , "COINBASE:PAIR-BTC-USD" + , "COINBASE:PAIR-DOT-USD" + , "COINBASE:PAIR-ETH-USD" + , "COINBASE:PAIR-LTC-USD" + , "COINBASE:PAIR-SOL-USD" + , "COINBASE:PAIR-XRP-USD" + ] + , "COINBASE_CVTTDATA": [ + "COINBASE:PAIR-ADA-USD" + , "COINBASE:PAIR-BCH-USD" + , "COINBASE:PAIR-BTC-USD" + , "COINBASE:PAIR-DOT-USD" + , "COINBASE:PAIR-ETH-USD" + , "COINBASE:PAIR-LTC-USD" + , "COINBASE:PAIR-SOL-USD" + , "COINBASE:PAIR-XRP-USD" + ] + } + , "db": { + "class": "cvttpy.tools.db.timescale.tsdb_client.TimescaleDbClient" + } + , "flush_interval_sec": 0.5 + + , "collections": { + "__dummy__": null + , "ALPACA": { + # dynamic instruments names + "tables": { + "trades": "alpaca_iex.md_trades" + , "booktops": "alpaca_iex.md_booktops" + , "booksnaps": "alpaca_iex.md_booksnaps" + } + } + , "BNBSPOT": { + "instruments": { + "PAIR-ADA-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-BCH-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-BTC-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-DOT-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-ETH-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-LTC-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-SOL-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-USDC-USDT": ["TRADES", "BOOK_TOP"] + , "PAIR-XRP-USDT": ["TRADES", "BOOK_TOP"] + } + , "tables": { + "trades": "bnbspot.md_trades" + , "booktops": "bnbspot.md_booktops" + , "booksnaps": "bnbspot.md_booksnaps" + } + } + , "BNBFUT": { + "instruments": { + "PERP-BTC-USDT": ["TRADES", "BOOK_TOP"] + , "PERP-ETH-USDT": ["TRADES", "BOOK_TOP"] + } + , "tables": { + "trades": "bnbfut.md_trades" + , "booktops": "bnbfut.md_booktops" + , "booksnaps": "bnbfut.md_booksnaps" + } + } + , "COINBASE": { + "instruments": { + "PAIR-ADA-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-BCH-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-BTC-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-DOT-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-ETH-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-LTC-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-SOL-USD": ["TRADES", "BOOK_TOP"] + , "PAIR-XRP-USD": ["TRADES", "BOOK_TOP"] + } + , "tables": { + "trades": "coinbase.md_trades" + , "booktops": "coinbase.md_booktops" + , "booksnaps": "coinbase.md_booksnaps" + } + } + } + } + + # ---------------------------------- + , "exchanges_settings": { + "*": { + "order_entry": { + "enabled": false + , "streaming": false + }, + "mkt_data": { + "enabled": true + , "streaming": true + }, + "accounting": { + "enabled": false + , "streaming": false + } + } + } +} + diff --git a/market_data/md_recorder/entrypoint.sh b/__DECO__/md_recorder/entrypoint.sh similarity index 100% rename from market_data/md_recorder/entrypoint.sh rename to __DECO__/md_recorder/entrypoint.sh diff --git a/market_data/md_recorder/requirements.txt b/__DECO__/md_recorder/requirements.txt similarity index 100% rename from market_data/md_recorder/requirements.txt rename to __DECO__/md_recorder/requirements.txt diff --git a/microservices/.env b/__DECO__/microservices/.env similarity index 97% rename from microservices/.env rename to __DECO__/microservices/.env index eefa14c..3e3a6d3 100644 --- a/microservices/.env +++ b/__DECO__/microservices/.env @@ -3,7 +3,7 @@ CONFIG_COMM_SETTINGS=comm/pubsub/redis/docker_microservices CONFIG_FILE="http://${CONFIG_SERVICE}/apps/cvtt_musvc" CVTT_USER=1001:1001 -CVTT_VERSION=0.0.3 +CVTT_VERSION=0.0.6 # MDGW_CREDS_FILE=.creds MDGW_LOG_FILE=%T.md_gateway.log diff --git a/__DECO__/microservices/VERSION b/__DECO__/microservices/VERSION new file mode 100644 index 0000000..5c4511c --- /dev/null +++ b/__DECO__/microservices/VERSION @@ -0,0 +1 @@ +0.0.7 \ No newline at end of file diff --git a/microservices/ac_gateway/.dockerignore b/__DECO__/microservices/ac_gateway/.dockerignore similarity index 100% rename from microservices/ac_gateway/.dockerignore rename to __DECO__/microservices/ac_gateway/.dockerignore diff --git a/microservices/ac_gateway/Dockerfile b/__DECO__/microservices/ac_gateway/Dockerfile similarity index 100% rename from microservices/ac_gateway/Dockerfile rename to __DECO__/microservices/ac_gateway/Dockerfile diff --git a/microservices/ac_gateway/entrypoint.sh b/__DECO__/microservices/ac_gateway/entrypoint.sh similarity index 100% rename from microservices/ac_gateway/entrypoint.sh rename to __DECO__/microservices/ac_gateway/entrypoint.sh diff --git a/microservices/ac_gateway/requirements.txt b/__DECO__/microservices/ac_gateway/requirements.txt similarity index 100% rename from microservices/ac_gateway/requirements.txt rename to __DECO__/microservices/ac_gateway/requirements.txt diff --git a/microservices/bookkeeper/.dockerignore b/__DECO__/microservices/bookkeeper/.dockerignore similarity index 100% rename from microservices/bookkeeper/.dockerignore rename to __DECO__/microservices/bookkeeper/.dockerignore diff --git a/microservices/bookkeeper/Dockerfile b/__DECO__/microservices/bookkeeper/Dockerfile similarity index 100% rename from microservices/bookkeeper/Dockerfile rename to __DECO__/microservices/bookkeeper/Dockerfile diff --git a/microservices/bookkeeper/entrypoint.sh b/__DECO__/microservices/bookkeeper/entrypoint.sh similarity index 100% rename from microservices/bookkeeper/entrypoint.sh rename to __DECO__/microservices/bookkeeper/entrypoint.sh diff --git a/microservices/bookkeeper/requirements.txt b/__DECO__/microservices/bookkeeper/requirements.txt similarity index 100% rename from microservices/bookkeeper/requirements.txt rename to __DECO__/microservices/bookkeeper/requirements.txt diff --git a/microservices/build.sh b/__DECO__/microservices/build.sh similarity index 76% rename from microservices/build.sh rename to __DECO__/microservices/build.sh index 30e7db9..001c568 100755 --- a/microservices/build.sh +++ b/__DECO__/microservices/build.sh @@ -2,20 +2,26 @@ # --- Settings RegistryService=cloud21.cvtt.vpn:5500 -ValidImages=(md_gateway) -ValidImages=(oe_gateway) -ValidImages=(ac_gateway) -ValidImages=(pricer) -ValidImages=(coordinator) -ValidImages=(bookkeeper) + +ValidImages=() +ValidImages+=(md_gateway) +ValidImages+=(oe_gateway) +ValidImages+=(ac_gateway) +ValidImages+=(pricer) +ValidImages+=(coordinator) +ValidImages+=(bookkeeper) ValidImages+=(risk_manager) ValidImages+=(quant) ValidImages+=(trader) -BashFunctionCode=$(realpath $(dirname ${0}))/../dckr_bld_functions.sh + +ScriptPath=$(realpath $(dirname ${0})) +BashFunctionCode=${ScriptPath}/../dckr_bld_functions.sh +ImageVersion=$(cat ${ScriptPath}/VERSION) + # --- Settings usage() { - echo "Usage: $0 -I -V -L :" + echo "Usage: $0 -I -L : [-V ] " exit 1 } @@ -68,8 +74,9 @@ build_musvc_image() { Project=${2} ProjectVersion=${3} - DockerDir=$(realpath $(dirname ${0})/${ImageName}) - cd ${DockerDir} + DockerDir=${ScriptPath}/${ImageName} + Cmd="cd ${DockerDir}" + echo ${Cmd} && eval ${Cmd} if [ "${Project}" != "" ]; then get_project_version ${Project} ${ProjectVersion} @@ -100,7 +107,6 @@ for ImageName in "${Images[@]}"; do build_musvc_image ${ImageName} ${Project} ${ProjectVersion} done -# echo "**** D E B U G E X I T" && exit echo "***** ${0} D O N E" diff --git a/microservices/coordinator/.dockerignore b/__DECO__/microservices/coordinator/.dockerignore similarity index 100% rename from microservices/coordinator/.dockerignore rename to __DECO__/microservices/coordinator/.dockerignore diff --git a/microservices/coordinator/Dockerfile b/__DECO__/microservices/coordinator/Dockerfile similarity index 100% rename from microservices/coordinator/Dockerfile rename to __DECO__/microservices/coordinator/Dockerfile diff --git a/microservices/coordinator/entrypoint.sh b/__DECO__/microservices/coordinator/entrypoint.sh similarity index 60% rename from microservices/coordinator/entrypoint.sh rename to __DECO__/microservices/coordinator/entrypoint.sh index f92d423..891f1dd 100755 --- a/microservices/coordinator/entrypoint.sh +++ b/__DECO__/microservices/coordinator/entrypoint.sh @@ -3,6 +3,15 @@ LogDir=/logs Cmd="python3.12" + +if [ "${COORD_DEBUG}" == "Y" ]; then + Cmd+=" -m debugpy" + Cmd+=" --listen 0.0.0.0:5678" + if [ "${COORD_DEBUG_WAIT}" == "Y" ]; then + Cmd+=" --wait-for-client" + fi +fi + Cmd+=" cvttpy/apps/microservices/coordinator/coordinator.py" Cmd+=" --config=${CONFIG_FILE:-/config/cvtt_musvc.cfg}" diff --git a/microservices/trader/requirements.txt b/__DECO__/microservices/coordinator/requirements.txt similarity index 91% rename from microservices/trader/requirements.txt rename to __DECO__/microservices/coordinator/requirements.txt index e840a84..ceeb72d 100644 --- a/microservices/trader/requirements.txt +++ b/__DECO__/microservices/coordinator/requirements.txt @@ -7,3 +7,4 @@ python-dateutil>=2.8.2 types-python-dateutil>=2.8.19.6 cryptography>=43.0.0 PyJWT>=2.10.1 +debugpy>=1.8.13 diff --git a/microservices/docker-compose.yml b/__DECO__/microservices/docker-compose.yml similarity index 100% rename from microservices/docker-compose.yml rename to __DECO__/microservices/docker-compose.yml diff --git a/microservices/md_gateway/.dockerignore b/__DECO__/microservices/md_gateway/.dockerignore similarity index 100% rename from microservices/md_gateway/.dockerignore rename to __DECO__/microservices/md_gateway/.dockerignore diff --git a/microservices/md_gateway/Dockerfile b/__DECO__/microservices/md_gateway/Dockerfile similarity index 100% rename from microservices/md_gateway/Dockerfile rename to __DECO__/microservices/md_gateway/Dockerfile diff --git a/microservices/md_gateway/entrypoint.sh b/__DECO__/microservices/md_gateway/entrypoint.sh similarity index 100% rename from microservices/md_gateway/entrypoint.sh rename to __DECO__/microservices/md_gateway/entrypoint.sh diff --git a/microservices/coordinator/requirements.txt b/__DECO__/microservices/md_gateway/requirements.txt similarity index 100% rename from microservices/coordinator/requirements.txt rename to __DECO__/microservices/md_gateway/requirements.txt diff --git a/microservices/oe_gateway/.dockerignore b/__DECO__/microservices/oe_gateway/.dockerignore similarity index 100% rename from microservices/oe_gateway/.dockerignore rename to __DECO__/microservices/oe_gateway/.dockerignore diff --git a/microservices/oe_gateway/Dockerfile b/__DECO__/microservices/oe_gateway/Dockerfile similarity index 100% rename from microservices/oe_gateway/Dockerfile rename to __DECO__/microservices/oe_gateway/Dockerfile diff --git a/microservices/oe_gateway/entrypoint.sh b/__DECO__/microservices/oe_gateway/entrypoint.sh similarity index 100% rename from microservices/oe_gateway/entrypoint.sh rename to __DECO__/microservices/oe_gateway/entrypoint.sh diff --git a/microservices/oe_gateway/requirements.txt b/__DECO__/microservices/oe_gateway/requirements.txt similarity index 100% rename from microservices/oe_gateway/requirements.txt rename to __DECO__/microservices/oe_gateway/requirements.txt diff --git a/microservices/pricer/.dockerignore b/__DECO__/microservices/pricer/.dockerignore similarity index 100% rename from microservices/pricer/.dockerignore rename to __DECO__/microservices/pricer/.dockerignore diff --git a/microservices/pricer/Dockerfile b/__DECO__/microservices/pricer/Dockerfile similarity index 100% rename from microservices/pricer/Dockerfile rename to __DECO__/microservices/pricer/Dockerfile diff --git a/microservices/pricer/entrypoint.sh b/__DECO__/microservices/pricer/entrypoint.sh similarity index 100% rename from microservices/pricer/entrypoint.sh rename to __DECO__/microservices/pricer/entrypoint.sh diff --git a/microservices/md_gateway/requirements.txt b/__DECO__/microservices/pricer/requirements.txt similarity index 100% rename from microservices/md_gateway/requirements.txt rename to __DECO__/microservices/pricer/requirements.txt diff --git a/microservices/quant/.dockerignore b/__DECO__/microservices/quant/.dockerignore similarity index 100% rename from microservices/quant/.dockerignore rename to __DECO__/microservices/quant/.dockerignore diff --git a/microservices/quant/Dockerfile b/__DECO__/microservices/quant/Dockerfile similarity index 100% rename from microservices/quant/Dockerfile rename to __DECO__/microservices/quant/Dockerfile diff --git a/microservices/quant/entrypoint.sh b/__DECO__/microservices/quant/entrypoint.sh similarity index 100% rename from microservices/quant/entrypoint.sh rename to __DECO__/microservices/quant/entrypoint.sh diff --git a/microservices/pricer/requirements.txt b/__DECO__/microservices/quant/requirements.txt similarity index 100% rename from microservices/pricer/requirements.txt rename to __DECO__/microservices/quant/requirements.txt diff --git a/microservices/risk_manager/.dockerignore b/__DECO__/microservices/risk_manager/.dockerignore similarity index 100% rename from microservices/risk_manager/.dockerignore rename to __DECO__/microservices/risk_manager/.dockerignore diff --git a/microservices/risk_manager/Dockerfile b/__DECO__/microservices/risk_manager/Dockerfile similarity index 100% rename from microservices/risk_manager/Dockerfile rename to __DECO__/microservices/risk_manager/Dockerfile diff --git a/microservices/risk_manager/entrypoint.sh b/__DECO__/microservices/risk_manager/entrypoint.sh similarity index 100% rename from microservices/risk_manager/entrypoint.sh rename to __DECO__/microservices/risk_manager/entrypoint.sh diff --git a/microservices/quant/requirements.txt b/__DECO__/microservices/risk_manager/requirements.txt similarity index 100% rename from microservices/quant/requirements.txt rename to __DECO__/microservices/risk_manager/requirements.txt diff --git a/microservices/trader/.dockerignore b/__DECO__/microservices/trader/.dockerignore similarity index 100% rename from microservices/trader/.dockerignore rename to __DECO__/microservices/trader/.dockerignore diff --git a/microservices/trader/Dockerfile b/__DECO__/microservices/trader/Dockerfile similarity index 100% rename from microservices/trader/Dockerfile rename to __DECO__/microservices/trader/Dockerfile diff --git a/microservices/trader/entrypoint.sh b/__DECO__/microservices/trader/entrypoint.sh similarity index 100% rename from microservices/trader/entrypoint.sh rename to __DECO__/microservices/trader/entrypoint.sh diff --git a/microservices/risk_manager/requirements.txt b/__DECO__/microservices/trader/requirements.txt similarity index 100% rename from microservices/risk_manager/requirements.txt rename to __DECO__/microservices/trader/requirements.txt diff --git a/__DECO__/trading_simulator/Dockerfile b/__DECO__/trading_simulator/Dockerfile new file mode 100644 index 0000000..26381be --- /dev/null +++ b/__DECO__/trading_simulator/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.12-slim + +COPY requirements.txt / + +RUN pip install --upgrade pip --root-user-action=ignore +RUN pip install -r /requirements.txt --root-user-action=ignore + +COPY cvttpy /cvttpy + +RUN mkdir -p /logs /config /data + +SHELL ["/bin/bash", "-c"] + +ENV PYTHONPATH=/ + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/__DECO__/trading_simulator/VERSION b/__DECO__/trading_simulator/VERSION new file mode 100644 index 0000000..8a9ecc2 --- /dev/null +++ b/__DECO__/trading_simulator/VERSION @@ -0,0 +1 @@ +0.0.1 \ No newline at end of file diff --git a/__DECO__/trading_simulator/build.sh b/__DECO__/trading_simulator/build.sh new file mode 100755 index 0000000..f3ff95f --- /dev/null +++ b/__DECO__/trading_simulator/build.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +# --- Settings +RegistryService=cloud21.cvtt.vpn:5500 +ImageName=trading_simulator +BashFunctionCode=$(realpath $(dirname ${0}))/../dckr_bld_functions.sh +ImageVersion=$(cat $(realpath $(dirname ${0}))/VERSION) +# --- Settings + +usage() { + echo "Usage: $0 -L : [-V ] " + exit 1 +} + + +# ---------------- cmdline +while getopts "I:V:L:" opt; do + case ${opt} in + V ) + ImageVersion=$OPTARG + ;; + L ) + Library=$OPTARG + ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + usage + ;; + : ) + echo "Option -$OPTARG requires an argument." >&2 + usage + ;; + esac +done +# ---------------- cmdline + +if [ "${ImageName}" == "" ] +then + usage +fi + +if [ "${ImageVersion}" == "" ] +then + usage +fi + +DockerDir=$(realpath $(dirname ${0})/${ImageName}) +cd ${DockerDir} + +Project= +ProjectVersion= +if [ "${Library}" != "" ] ; then + IFS=':' read -ra parts <<< "${Library}" + Project=${parts[0]} + ProjectVersion=${parts[1]} +fi +# exit + +build_musvc_image() { + ImageName=${1} + Project=${2} + ProjectVersion=${3} + + DockerDir=$(realpath $(dirname ${0})) + cd ${DockerDir} + + if [ "${Project}" != "" ]; then + get_project_version ${Project} ${ProjectVersion} + fi + + function cleanup { + cd ${DockerDir} + rm -rf cvttpy + } + trap cleanup RETURN + + build_docker_image ${ImageName} ${RegistryService} ${ImageVersion} ${Project} ${ProjectVersion} +} + + +source ${BashFunctionCode} +build_musvc_image ${ImageName} ${Project} ${ProjectVersion} + +# echo "**** D E B U G E X I T" && exit +echo "***** ${0} D O N E" + + diff --git a/__DECO__/trading_simulator/deployment/.env b/__DECO__/trading_simulator/deployment/.env new file mode 100644 index 0000000..cbd54d7 --- /dev/null +++ b/__DECO__/trading_simulator/deployment/.env @@ -0,0 +1,9 @@ +TRDSIM_VERSION=0.0.1 + +CONFIG_SERVICE=cloud16.cvtt.vpn:6789 +CONFIG_FILE="/config/simulation.cfg" + +TRDSIM_USER=1001:1001 + +TRDSIM_LOG_FILE=%T.trading_simulator.log +TRDSIM_ADD_ARGS= diff --git a/__DECO__/trading_simulator/deployment/config/simulation.cfg b/__DECO__/trading_simulator/deployment/config/simulation.cfg new file mode 100644 index 0000000..cb5f270 --- /dev/null +++ b/__DECO__/trading_simulator/deployment/config/simulation.cfg @@ -0,0 +1,131 @@ +@var DATA_DIR=/home/oleg/develop/simulation/data + +@var FROM_TIME="2024-11-14 13:30:00" +@var INTERVAL="18 minutes" +# @var INTERVAL="5 minutes" + +{ + "refdata": @inc=http://@env{CONFIG_SERVICE}/refdata/refdata + , "comm": @inc=http://@env{CONFIG_SERVICE}/apps/common/comm + , "comm": @inc=http://@env{CONFIG_SERVICE}/apps/common/comm + , "comm_settings": "/comm/pubsub/redis/localhost" + , "simulation": { + # "speed_factor": 1.0, + "speed_factor": 0.0, + "time_events_interval_ms": 0 + } + , "market_simulator" : { + "initial_positions": { + "exchange": "COINBASE", + "assets": { + "BTC": 0.5 + , "ETH": 0.0 + , "USD": 10_000.0 + } + } + , "market_data": { + "sources": [ + { + "loader_class": "cvttpy.trading.mkt_data.time_series.loaders.sqlite_loader.SqliteMdLoader" + , "file_name": "${DATA_DIR}/20250422.CNBS.BTC_USD.db" + , "table_name": "md_trades" + , "columns": [ + "tstamp" + , "tstamp_ns as tm" # must have tm for DataFrame + , "tstamp_ns as local_time_ns" # must have for simrunner + , "exchange_id" + , "instrument_id" + , "px" + , "qty" + , "trade_id" + , "condition as agressor_side" + ] + , "time_column": "tstamp" + , "exchange_id": "COINBASE" + , "instruments": ["PAIR-BTC-USD"] + , "md_type": "TRADES" + } + # , { + # "loader_class": "cvttpy.trading.mkt_data.time_series.loaders.tsdb_loader.TimescaleTSLoader" + # , "credentials_key": "TSDB_MD_CVTTDATA_RO" + # , "interval": ${INTERVAL} + # , "from_time": ${FROM_TIME} #"2024-10-03 09:00:00" + # , "exchange_id": "COINBASE" + # , "instruments": ["PAIR-BTC-USD"] + # , "md_type": "BOOK_TOP" + # , "table_name": "coinbase.md_booktops" + # , "columns": [ + # "time", + # "local_time_ns", + # "exchange_id", + # "instrument_id", + # "bid_price", + # "bid_quantity", + # "ask_price", + # "ask_quantity", + # "'BT' as type", + # ] + # , "time_column": "time" + # } + ] + , "api": { + "web_socket": { + "url": "ws://localhost:8080/ws" + } + } + } + , "order_entry": { + "execution": { + "execute_on": ["BOOK_TOP", "TRADES"] + "top_only": true + ,"min_order_age_sec": 0.1 + , "pricing_method": "BY_MKT_TRD_PRICE_BY_TRADE" + + , "short_sale_allowed": false + , "fee_schedule": { + "taker_fee_rate": 0.006 + "maker_fee_rate": 0.004 + } + # , "presubscribe_md_types": ["BOOK_TOP", "TRADES"] + } + , "api": { + "web_socket": { + "url": "ws://localhost:8180/ws" + } + , "REST": { + "port": 8181 + "new_order_url": ["POST", "http://localhost:8181/order"] + , "cancel_order_url": ["POST", "http://localhost:8181/cancel"] + , "cancel_all_url": ["POST", "http://localhost:8181/cancel_all"] + , "open_orders_url": ["GET", "http://localhost:8181/open_orders"] + , "order_status_url": ["GET", "http://localhost:8181/order_status"] + } + } + } + , "accounting": { + "api": { + "REST": { + "port": 8182 + , "positions_url": ["GET", "http://localhost:8182/positions"] + , "trades_url": ["GET", "http://localhost:8182/trades"] + } + } + } + } + , "external_participants": [ + { + "class": "cvttpy.apps.trading_simulator.participants.external_strategy.ExternalStrategySim" + "params": { + "reader": { + "source": "file:///home/oleg/develop/cvtt2/simulation/trading_instructions.json" + } + "listener": { + # communicate with coordinator + "coordinator_url": "http://localhost:12345" + , "book_id": "TESTBK_COINBS_01" + , "strategy_id": "DUMMY_STRATEGY" + } + } + } + ] +} diff --git a/__DECO__/trading_simulator/deployment/docker-compose.yml b/__DECO__/trading_simulator/deployment/docker-compose.yml new file mode 100644 index 0000000..f67dc63 --- /dev/null +++ b/__DECO__/trading_simulator/deployment/docker-compose.yml @@ -0,0 +1,16 @@ +# CVTT Microservices +services: + trdsim: + image: cloud21.cvtt.vpn:5500/trading_simulator:${TRDSIM_VERSION} + container_name: trading_simulator + pull_policy: always + user: ${TRDSIM_USER:-1001:1001} + environment: + CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789} + CONFIG_FILE: ${CONFIG_FILE:-http://${CONFIG_SERVICE}/apps/cvtt_musvc} + TRDSIM_LOG_FILE: ${TRDSIM_LOG_FILE:-%T.trader.log} + TRDSIM_ADD_ARGS: ${TRDSIM_ADD_ARGS:-} + volumes: + - ./config:/config + - ./data:/data + - ./logs:/logs diff --git a/__DECO__/trading_simulator/entrypoint.sh b/__DECO__/trading_simulator/entrypoint.sh new file mode 100755 index 0000000..fc5940b --- /dev/null +++ b/__DECO__/trading_simulator/entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +LogDir=/logs + +Cmd="python3.12" +Cmd+=" cvttpy/apps/trading_simulator/main.py" + +Cmd+=" --config=${CONFIG_FILE:-/config/simulation.cfg}" +Cmd+=" --simtime_file=${SIMTIME_FILE:-/data/simtime.bin}" +Cmd+=" --cfgvar=DATA_DIR=/data" + +Cmd+=" --log_file=${LogDir}/${TRDSIM_LOG_FILE:-%T.trading_simulator.log}" +Cmd+=" ${TRDSIM_ADD_ARGS}" +echo ${Cmd} +exec ${Cmd} + diff --git a/__DECO__/trading_simulator/requirements.txt b/__DECO__/trading_simulator/requirements.txt new file mode 100644 index 0000000..975615c --- /dev/null +++ b/__DECO__/trading_simulator/requirements.txt @@ -0,0 +1,10 @@ +aiohttp>=3.7.4.post0 +nest-asyncio>=1.5.5 +hjson>=3.1.0 +sortedcontainers>=2.4.0 +pandas>=1.5.3 +# redis>=5.0.8 +# python-dateutil>=2.8.2 +# types-python-dateutil>=2.8.19.6 +# cryptography>=43.0.0 +# PyJWT>=2.10.1 diff --git a/test/rsync_test/Dockerfile b/test/rsync_test/Dockerfile deleted file mode 100644 index 6fd40a9..0000000 --- a/test/rsync_test/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM python:3.12-slim - -ARG FROM_DIR=docker_dev/test/rsync_test - -COPY ${FROM_DIR}/test.sh /test.sh - -RUN apt-get update && apt-get install -y rsync openssh-client - -COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa -COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub - -RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub - -# Shared Volumes -RUN chmod +x /test.sh - -WORKDIR / -SHELL ["/bin/bash", "-c"] -ENV PYTHONPATH=/ -ENTRYPOINT [ "/test.sh" ] - diff --git a/test/rsync_test/test.sh b/test/rsync_test/test.sh deleted file mode 100755 index f388ee1..0000000 --- a/test/rsync_test/test.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -date > /testfile.txt - -Source=/testfile.txt -Targets= -Targets="${Targets} cvtt@cloud21.cvtt.vpn:/tmp/" -Targets="${Targets} cvtt@hs01.cvtt.vpn:/tmp/" - - -for tgt in ${Targets} -do - Cmd="/usr/bin/rsync -ahv" - Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'" - Cmd+=" ${Source} ${tgt}" - echo $Cmd - eval $Cmd -done -