From 8b109e933c01cb178ca973b0dcacfc02132e8bb0 Mon Sep 17 00:00:00 2001 From: Cryptoval Trading Technologies Date: Tue, 3 Dec 2024 18:58:02 +0000 Subject: [PATCH] progress --- data/admin/cvtt_hosts.cfg | 147 +++++++++++--------- data/apps/research/mkt_sim_client.cfg | 16 ++- data/apps/research/mkt_simulator_eqt.cfg | 164 +++++++++++------------ data/refdata/exchanges.cfg | 32 +++++ 4 files changed, 211 insertions(+), 148 deletions(-) diff --git a/data/admin/cvtt_hosts.cfg b/data/admin/cvtt_hosts.cfg index 83f2a33..15b4b95 100644 --- a/data/admin/cvtt_hosts.cfg +++ b/data/admin/cvtt_hosts.cfg @@ -1,153 +1,172 @@ +# ------- I M P O R T A N T +# ----------------------------------------------------------------- +# ---- DO NOT USE JSON Booleans, jq considers "false" as missing, +# ---- So it will replace it with default value +# ---- Instead use strings (case insensitive): "false", "no", "N", "T" +# ----------------------------------------------------------------- { "cvtt.vpn": { // ----------- C l o u d "cloud11": { "type": "cloud", "users": ["oleg", "cvtt"], - "docker": true, + "docker": "true", }, "cloud15": { "type": "cloud", "users": ["oleg"], - "docker": false, + "docker": "false", }, "cloud16": { "type": "cloud", "users": ["oleg", "cvtt"], - "docker": true, + "docker": "true", }, "cloud21": { "type": "cloud", "users": ["oleg", "cvtt"], - "docker": true, + "docker": "true", }, "cloud22": { "type": "cloud", "users": ["oleg"], - "docker": false, + "docker": "false", }, "cloud23": { "type": "cloud", "users": ["oleg", "cvtt"], - "docker": true, + "docker": "true", }, "cloud24": { "type": "cloud", "users": ["oleg", "cvtt"], - "docker": true, + "docker": "true", }, // ----------- B a r e M e t a l "homestore": { "type": "metal", "users": ["oleg"], - "docker": true, + "docker": "true", }, "nsbackup": { "type": "metal", "users": ["oleg"], - "docker": false, + "docker": "false", }, "cryptoval2": { "type": "metal", "users": ["oleg"], - "docker": false, - "desktop": true, + "docker": "false", + "desktop": "true", }, "cryptoval3": { "type": "metal", "users": ["oleg", "cvtt"], - "docker": true, - "desktop": true, + "docker": "true", + "desktop": "true", }, "cryptoval4": { "type": "metal", "users": ["oleg", "cvtt"], - "docker": true, - "desktop": true, + "docker": "true", + "desktop": "true", }, "dtvmhost": { "type": "metal", "users": ["oleg"], - "docker": false, - "desktop": true, + "docker": "false", + "desktop": "true", }, "gpushnik": { "type": "metal", "users": ["oleg"], - "docker": true, + "docker": "true", }, // ----------- V i r t u a l M a c h i n e s + // - GUI "cvtt-admin": { "type": "vm", "users": ["oleg"], - "docker": true, - "desktop": true, - }, - "cvtt-build": { - "type": "vm", - "users": ["oleg"], - "docker": true, + "docker": "true", + "gui": "true", }, "cvtt-dev": { "type": "vm", "users": ["oleg"], - "docker": true, - "desktop": true, + "docker": "true", + "gui": "true", }, "cvtt-edu": { "type": "vm", "users": ["oleg"], - "docker": false, - "desktop": true, - }, - "cvtt-md": { - "type": "vm", - "users": ["oleg", "cvtt"], - "docker": true, - }, - "cvtt-prod-03": { - "type": "vm", - "users": ["oleg"], - "docker": false, - "supervisor": true, - }, - "cvtt-prod-eqty-01": { - "type": "vm", - "users": ["oleg", "cvtt"], - "docker": true, + "docker": "false", + "gui": "true", }, "cvtt-research": { "type": "vm", "users": ["oleg"], - "docker": true, - }, - "cvtt-testserver": { - "type": "vm", - "users": ["oleg", "cvtt"], - "docker": true, - }, - "cvttdata": { - "type": "vm", - "users": ["oleg", "cvtt"], - "docker": true, - }, - "cvttdata-testdb": { - "type": "vm", - "users": ["oleg", "cvtt"], - "docker": true, + "docker": "true", + "gui": "true", }, "desktop": { "type": "vm", "users": ["oleg"], - "docker": false, - "desktop": true, + "docker": "false", + "gui": "true", + }, + + // -- Headless + "cvtt-build": { + "type": "vm", + "users": ["oleg"], + "docker": "true", + }, + "cvtt-md": { + "type": "vm", + "users": ["oleg", "cvtt"], + "docker": "true", + }, + "cvtt-prod-03": { + "type": "vm", + "users": ["oleg"], + "docker": "false", + "supervisor": "true", + }, + "cvtt-prod-eqty-01": { + "type": "vm", + "users": ["oleg", "cvtt"], + "docker": "true", + }, + "cvttdata": { + "type": "vm", + "users": ["oleg", "cvtt"], + "docker": "true", + }, + "cvtt-testserver": { + "type": "vm", + "users": ["oleg", "cvtt"], + "docker": "true", + "to_check": "No", + }, + "cvttdata-testdb": { + "type": "vm", + "users": ["oleg", "cvtt"], + "docker": "true", + "to_check": "false", + "timeout_sec": 11, }, "ops-server": { "type": "vm", "users": ["oleg", "cvtt"], - "docker": true, + "docker": "true", + }, + "cvtt-dev-server": { + "type": "vm", + "users": ["oleg", "cvtt"], + "docker": "true", + "to_check": "Yes", }, }, diff --git a/data/apps/research/mkt_sim_client.cfg b/data/apps/research/mkt_sim_client.cfg index 017024e..51a4524 100644 --- a/data/apps/research/mkt_sim_client.cfg +++ b/data/apps/research/mkt_sim_client.cfg @@ -29,7 +29,21 @@ "enabled": true , "streaming": false }, - } + }, + "ALPACA_SIM": { + "mkt_data": { + "enabled": true + , "streaming": true + }, + "order_entry": { + "enabled": true + , "streaming": true + }, + "accounting": { + "enabled": true + , "streaming": false + }, + }, } } diff --git a/data/apps/research/mkt_simulator_eqt.cfg b/data/apps/research/mkt_simulator_eqt.cfg index 5d2edf9..e0dd677 100644 --- a/data/apps/research/mkt_simulator_eqt.cfg +++ b/data/apps/research/mkt_simulator_eqt.cfg @@ -1,10 +1,13 @@ @var CONFIG_SERVICE=cloud23.cvtt.vpn:6789 -@var SQLITE_DIR=/home/oleg/tmp/sqlite -@var DATE="20241114" -# @var FROM_TIME="2024-11-14 13:30:00" -# @var INTERVAL="18 minutes" -# @var INTERVAL="5 minutes" +@var DATA_DIR=/home/coder/project/data + +@var MD_WS_PORT=10180 +@var OE_WS_PORT=10181 +@var OE_REST_PORT=10182 +@var ACCT_REST_PORT=10083 + +@var DATE=20241114 { "refdata": @inc=http://${CONFIG_SERVICE}/refdata/refdata @@ -29,75 +32,70 @@ "sources": [ { "loader_class": "cvttpy.trading.mkt_data.time_series.loaders.sqlite_loader.SqliteMdLoader" - , "file_name": "${SQLITE_DIR}/20240906.COIN.1min.db" - , "table_name": "md_1min_bars" - , "columns": [ - "tstamp" - , "tstamp_ns as tm" # must have tm for DataFrame - , "tstamp_ns as local_time_ns" # must have for simrunner - , "exchange_id" - , "instrument_id" - , "open" - , "high" - , "low" - , "close" - , "volume" - , "vwap" - , "num_trades" - ] - , "time_column": "tstamp" + , "file_name": "${DATA_DIR}/eqty/${DATE}.alpaca_sim_md.db" , "exchange_id": "ALPACA" - , "instruments": ["STOCK-COIN"] - , "md_type": "HIST_BAR" + , "instruments": ["STOCK-COIN", "STOCK-CAN"] + , "md_types": { + "TRADES": { + "table_name": "md_trades" + , "columns": [ + "tstamp" + , "tstamp_ns as tm" # required column + , "tstamp_ns as exchange_ts_ns" + , "tstamp_ns as local_time_ns" + , "exchange_id" + , "instrument_id" + , "exch as exchange_code" + , "px as price" + , "qty as quantity" + , "'' as taker_side" # get from condition (?) + , "trade_id" + , "condition" + , "tape" + ] + , "time_column": "tstamp" + } + , "BOOK_TOP": { + "table_name": "md_quotes" + , "columns": [ + "tstamp" + , "tstamp_ns as tm" # required column + , "tstamp_ns as local_time_ns" + , "exchange_id" + , "instrument_id" + , "bid_exch as bid_exchange_code" + , "ask_exch as ask_exchange_code" + , "bid_px as bid_price" + , "bid_qty as bid_quantity" + , "ask_px as ask_price" + , "ask_qty as ask_quantity" + ] + , "time_column": "tstamp" + } + # , "HIST_BAR": { + # "table_name": "md_1min_bars" + # , "columns": [ + # "tstamp" + # , "tstamp_ns as tm" # must have tm for DataFrame + # , "tstamp_ns as local_time_ns" # must have for simrunner + # , "exchange_id" + # , "instrument_id" + # , "open" + # , "high" + # , "low" + # , "close" + # , "volume" + # , "vwap" + # , "num_trades" + # ] + # , "time_column": "tstamp" + # } + } } - # { - # "loader_class": "cvttpy.trading.mkt_data.time_series.loaders.tsdb_loader.TimescaleTSLoader" - # , "credentials_key": "TSDB_MD_CVTTDATA_RO" - # , "interval": ${INTERVAL} - # , "from_time": ${FROM_TIME} #"2024-10-03 09:00:00" - # , "exchange_id": "COINBASE" - # , "instruments": ["PAIR-BTC-USD"] - # , "md_type": "TRADES" - # , "table_name": "coinbase.md_trades" - # , "columns": [ - # "time", - # "local_time_ns", - # "exchange_ts_ns", - # "exchange_id", - # "instrument_id", - # "price", - # "quantity", - # "taker_side", - # "'TRD' as type", - # ] - # , "time_column": "time" - # } - # , { - # "loader_class": "cvttpy.trading.mkt_data.time_series.loaders.tsdb_loader.TimescaleTSLoader" - # , "credentials_key": "TSDB_MD_CVTTDATA_RO" - # , "interval": ${INTERVAL} - # , "from_time": ${FROM_TIME} #"2024-10-03 09:00:00" - # , "exchange_id": "COINBASE" - # , "instruments": ["PAIR-BTC-USD"] - # , "md_type": "BOOK_TOP" - # , "table_name": "coinbase.md_booktops" - # , "columns": [ - # "time", - # "local_time_ns", - # "exchange_id", - # "instrument_id", - # "bid_price", - # "bid_quantity", - # "ask_price", - # "ask_quantity", - # "'BT' as type", - # ] - # , "time_column": "time" - # } - # ] + ] , "api": { "web_socket": { - "url": "ws://localhost:8080/ws" + "url": "ws://localhost:${MD_WS_PORT}/ws" } } } @@ -109,32 +107,32 @@ , "pricing_method": "BY_MKT_TRD_PRICE_BY_TRADE" , "short_sale_allowed": false - , "fee_schedule": { - "taker_fee_rate": 0.006 - "maker_fee_rate": 0.004 + , "fee_schedule": { # ALPACA - + "taker_fee_rate": 0.0 + "maker_fee_rate": 0.0 } # , "presubscribe_md_types": ["BOOK_TOP", "TRADES"] } , "api": { "web_socket": { - "url": "ws://localhost:8180/ws" + "url": "ws://localhost:${OE_WS_PORT}/ws" } , "REST": { - "port": 8181 - "new_order_url": ["POST", "http://localhost:8181/order"] - , "cancel_order_url": ["POST", "http://localhost:8181/cancel"] - , "cancel_all_url": ["POST", "http://localhost:8181/cancel_all"] - , "open_orders_url": ["GET", "http://localhost:8181/open_orders"] - , "order_status_url": ["GET", "http://localhost:8181/order_status"] + "port": ${OE_REST_PORT} + "new_order_url": ["POST", "http://localhost:${OE_REST_PORT}/order"] + , "cancel_order_url": ["POST", "http://localhost:${OE_REST_PORT}/cancel"] + , "cancel_all_url": ["POST", "http://localhost:${OE_REST_PORT}/cancel_all"] + , "open_orders_url": ["GET", "http://localhost:${OE_REST_PORT}/open_orders"] + , "order_status_url": ["GET", "http://localhost:${OE_REST_PORT}/order_status"] } } } , "accounting": { "api": { "REST": { - "port": 8182 - , "positions_url": ["GET", "http://localhost:8182/positions"] - , "trades_url": ["GET", "http://localhost:8182/trades"] + "port": ${ACCT_REST_PORT} + , "positions_url": ["GET", "http://localhost:${ACCT_REST_PORT}/positions"] + , "trades_url": ["GET", "http://localhost:${ACCT_REST_PORT}/trades"] } } } diff --git a/data/refdata/exchanges.cfg b/data/refdata/exchanges.cfg index 0682d7f..f2ab607 100644 --- a/data/refdata/exchanges.cfg +++ b/data/refdata/exchanges.cfg @@ -3,6 +3,38 @@ { "__DUMMY__": {} # ----------------------------- A + , "ALPACA_SIM": { + # https://docs.alpaca.markets/reference/ + "exchange_id": "ALPACA" + , "name": "Alpaca Trading" + , "type": "EQUITY" + + + , "mktdata" : { + "class": "cvttpy.exchanges.mkt_simulator.md_client.MktSimMdClientFeed" + "ws_url": ["WS", "ws://localhost:10180/ws"] + } + + , "order_entry": { + "class": "cvttpy.exchanges.mkt_simulator.oe_client.MarketSimulatorOEClient" + + , "REST": { + "new_order_url": ["POST", "http://localhost:10182/order"] + , "cancel_order_url": ["POST", "http://localhost:10182/cancel"] + , "cancel_all_url": ["POST", "http://localhost:10182/cancel_all"] + , "open_orders_url": ["GET", "http://localhost:10182/open_orders"] + , "order_status_url": ["GET", "http://localhost:10182/order_status"] + }, + "ws_url": ["WSS", "ws://localhost:10181/ws"] + } + , "accounting": { + "class": "cvttpy.exchanges.mkt_simulator.acct_client.MktSimAccounting" + , "REST": { + "positions_url": ["GET", "http://localhost:10083/positions"] + , "trades_url": ["GET", "http://localhost:10083/trades"] + } + } + } , "ALPACA_SNDBX": { # https://docs.alpaca.markets/reference/ "exchange_id": "ALPACA"