From d4d95f19fc50026dcf741c81e435b4d1da4ca544 Mon Sep 17 00:00:00 2001 From: Cryptoval Trading Technologies Date: Fri, 6 Sep 2024 20:03:38 +0000 Subject: [PATCH] progress --- data/admin/cvtt_hosts.cfg | 18 +- data/admin/git_repositories.cfg | 15 +- data/apps/active_instruments.cfg | 2 +- data/apps/algo_trade.cfg | 39 +++++ data/apps/common/redis.cfg | 5 - data/apps/cvtt.cfg | 39 +++++ data/apps/cvtt_crypto_coinbs.cfg | 39 +++++ data/apps/cvtt_eqt_alpaca.cfg | 7 +- data/apps/cvtt_eqt_alpaca_ilya.cfg | 180 -------------------- data/apps/md_rec_monitor.cfg | 4 +- data/apps/md_recorder.cfg | 6 +- data/apps/tests/listen_market_data_prod.cfg | 2 +- data/refdata/exchanges.cfg | 11 -- 13 files changed, 153 insertions(+), 214 deletions(-) delete mode 100644 data/apps/cvtt_eqt_alpaca_ilya.cfg diff --git a/data/admin/cvtt_hosts.cfg b/data/admin/cvtt_hosts.cfg index 36ff13b..6b0c928 100644 --- a/data/admin/cvtt_hosts.cfg +++ b/data/admin/cvtt_hosts.cfg @@ -108,12 +108,6 @@ "docker": false, "supervisor": false, }, - "cvtt-prod-01": { - "type": "vm", - "users": ["oleg", "cvtt"], - "docker": true, - "supervisor": false, - }, "cvtt-prod-eqty-01": { "type": "vm", "users": ["oleg", "cvtt"], @@ -150,6 +144,18 @@ "docker": true, "supervisor": false, }, + "cvtt-testserver": { + "type": "vm", + "users": ["oleg", "cvtt"], + "docker": true, + "supervisor": false, + }, + "cvtt-prod-03": { + "type": "vm", + "users": ["oleg"], + "docker": false, + "supervisor": true, + }, }, "cryptovaltrading.com": { diff --git a/data/admin/git_repositories.cfg b/data/admin/git_repositories.cfg index 31193b5..dc17d3e 100644 --- a/data/admin/git_repositories.cfg +++ b/data/admin/git_repositories.cfg @@ -42,8 +42,8 @@ {"homestore": "git@homestore.cvtt.vpn:/opt/git/cvtt2/jupyter_gpu.git" }, ], } - }, - "allbright": { + } + , "allbright": { "allbright-ops": [ {"origin": "ssh://allbrigh@allbrightacademy.com:7822/home/allbrigh/git/allbright-ops.git"}, {"homestore": "ssh://oleg@homestore.sheynin.home:/opt/git/allbright/allbright-ops.git"}, @@ -53,7 +53,7 @@ {"homestore": "ssh://oleg@homestore.sheynin.home:/opt/git/allbright/allbright.git"}, ] } - "devel-env": { + , "devel-env": { "cvtt-dev": [ {"gitea": "ssh://git@homestore.cvtt.vpn:20022/develop_env/.vscode.cvtt-dev.git"}, ] @@ -64,6 +64,15 @@ {"gitea": "ssh://git@homestore.cvtt.vpn:20022/develop_env/.vscode.oleg-macpro.git"}, ] }, + , "education-materials": { + "Algorithmic Trading Book": [ + {"origin": "ssh://git@homestore.cvtt.vpn:20022/edu/Algorithmic_Trading_Book.git"} + ] + + , "Kubernetes Cookbook": [ + {"origin": "ssh://git@homestore.cvtt.vpn:20022/edu/Kubernetes_Cookbook.git"} + ] + } "personal": { "obsidian_notes": [ {"origin": "ssh://cryptova@cryptovaltrading.com:1588/home/cryptova/git/personal/obsidian_notes.git"} diff --git a/data/apps/active_instruments.cfg b/data/apps/active_instruments.cfg index 707f8e6..ad10bfc 100644 --- a/data/apps/active_instruments.cfg +++ b/data/apps/active_instruments.cfg @@ -3,7 +3,7 @@ "dynamic_stock_list": { "exchange_id": "ALPACA" , "list_source": "redis" - , "redis_settings": "cvtt-prod-02" + , "redis_settings": "cvtt-prod-eqty-01" , "redis_key": "DAY_STOCK_LIST" } } diff --git a/data/apps/algo_trade.cfg b/data/apps/algo_trade.cfg index 6695a01..c5573d1 100644 --- a/data/apps/algo_trade.cfg +++ b/data/apps/algo_trade.cfg @@ -13,6 +13,45 @@ , "http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS&start={start}&end={end}" ] } + ###### N E W S E C T I O N (example) + , "simulation": { + "speed_factor": 0.0, + "time_events_interval_ms": 100 + } + , "market_simulator" : { + "market_data": { + "sources": [ + { + "loader_class": "cvttpy.research.time_series.loaders.tsdb_loader.TimescaleTSLoader" + , "credentials_key": "TSDB_MD_CVTTDATA_RO" + # , "interval": "4 hours" + , "interval": "2 hours" + , "from_time": "2024-04-15 09:00:00" + } + # , { + # "loader_class": "cvttpy.research.time_series.loaders.sqlite_loader.SqliteMdLoader", + # "file_name": "${SourcePath}/20240424.ALPACA.QAT.db" + # } + ] + } + , "accounting": { + "initial_positions": { + "COINBASE": { + "BTC": 0 + , "USD": 1000.0 + } + } + } + , "order_entry": { + "execution": { + "top_only": true + ,"min_order_age_sec": 0.1 + , "pricing_method": "BY_MKT_TRD_PRICE_BY_ORDER" + # , "presubscribe_md_types": ["BOOK_TOP", "TRADES"] + } + } + } + ###### N E W S E C T I O N - end , "backtest": { "source": { "loader_class": "cvttpy.research.time_series.loaders.sqlite_loader.SqliteMdLoader" diff --git a/data/apps/common/redis.cfg b/data/apps/common/redis.cfg index 3890607..5e20e2d 100644 --- a/data/apps/common/redis.cfg +++ b/data/apps/common/redis.cfg @@ -14,11 +14,6 @@ , "read_timeout" : 0.01 , "ping_timeout_secs": 2.0 } - , "cvtt-prod-02": { - "url": "redis://cvtt-prod-02.cvtt.vpn" - , "read_timeout" : 0.01 - , "ping_timeout_secs": 2.0 - } , "cvtt-prod-03": { "url": "redis://cvtt-prod-03.cvtt.vpn" , "read_timeout" : 0.01 diff --git a/data/apps/cvtt.cfg b/data/apps/cvtt.cfg index 63e4f0b..c59f1c4 100644 --- a/data/apps/cvtt.cfg +++ b/data/apps/cvtt.cfg @@ -13,6 +13,45 @@ , "books": { "CVTT_BK01": {} } + ###### N E W S E C T I O N (example) + , "simulation": { + "speed_factor": 0.0, + "time_events_interval_ms": 100 + } + , "market_simulator" : { + "market_data": { + "sources": [ + { + "loader_class": "cvttpy.research.time_series.loaders.tsdb_loader.TimescaleTSLoader" + , "credentials_key": "TSDB_MD_CVTTDATA_RO" + # , "interval": "4 hours" + , "interval": "2 hours" + , "from_time": "2024-04-15 09:00:00" + } + # , { + # "loader_class": "cvttpy.research.time_series.loaders.sqlite_loader.SqliteMdLoader", + # "file_name": "${SourcePath}/20240424.ALPACA.QAT.db" + # } + ] + } + , "accounting": { + "initial_positions": { + "COINBASE": { + "BTC": 0 + , "USD": 1000.0 + } + } + } + , "order_entry": { + "execution": { + "top_only": true + ,"min_order_age_sec": 0.1 + , "pricing_method": "BY_MKT_TRD_PRICE_BY_ORDER" + # , "presubscribe_md_types": ["BOOK_TOP", "TRADES"] + } + } + } + ###### N E W S E C T I O N - end , "backtest": { "exchanges_settings": { "*": { diff --git a/data/apps/cvtt_crypto_coinbs.cfg b/data/apps/cvtt_crypto_coinbs.cfg index 74d03af..6078398 100644 --- a/data/apps/cvtt_crypto_coinbs.cfg +++ b/data/apps/cvtt_crypto_coinbs.cfg @@ -13,6 +13,45 @@ , "books": { "CVTT_BK01": {} } + ###### N E W S E C T I O N (example) + , "simulation": { + "speed_factor": 0.0, + "time_events_interval_ms": 100 + } + , "market_simulator" : { + "market_data": { + "sources": [ + { + "loader_class": "cvttpy.research.time_series.loaders.tsdb_loader.TimescaleTSLoader" + , "credentials_key": "TSDB_MD_CVTTDATA_RO" + # , "interval": "4 hours" + , "interval": "2 hours" + , "from_time": "2024-04-15 09:00:00" + } + # , { + # "loader_class": "cvttpy.research.time_series.loaders.sqlite_loader.SqliteMdLoader", + # "file_name": "${SourcePath}/20240424.ALPACA.QAT.db" + # } + ] + } + , "accounting": { + "initial_positions": { + "COINBASE": { + "BTC": 0 + , "USD": 1000.0 + } + } + } + , "order_entry": { + "execution": { + "top_only": true + ,"min_order_age_sec": 0.1 + , "pricing_method": "BY_MKT_TRD_PRICE_BY_ORDER" + # , "presubscribe_md_types": ["BOOK_TOP", "TRADES"] + } + } + } + ###### N E W S E C T I O N - end , "backtest": { "exchanges_settings": { "*": { diff --git a/data/apps/cvtt_eqt_alpaca.cfg b/data/apps/cvtt_eqt_alpaca.cfg index 571e122..67a0ec1 100644 --- a/data/apps/cvtt_eqt_alpaca.cfg +++ b/data/apps/cvtt_eqt_alpaca.cfg @@ -161,9 +161,8 @@ } } , "trading_recorder": { - # "format": "CSV" - # "format": "TSDB" - "format": "SQLite" + "app_channel": "TRDREC" + , "format": "SQLite" # formats: "CSV", "TSDB", "SQLite" , "csv_settings": { "order_events_file_path": "%T.order_events.csv" "position_events_file_path": "%T.position_events.csv" @@ -179,7 +178,7 @@ } , "sqlite_settings": { # "file_name": "/tmp/%T.cvtt_backtest.db" - "file_name": "/tmp/%T.cvtt_alpaca_sandbox.db" + "file_name": "/logs/%T.cvtt_alpaca_sandbox.db" } } } diff --git a/data/apps/cvtt_eqt_alpaca_ilya.cfg b/data/apps/cvtt_eqt_alpaca_ilya.cfg deleted file mode 100644 index f707a3e..0000000 --- a/data/apps/cvtt_eqt_alpaca_ilya.cfg +++ /dev/null @@ -1,180 +0,0 @@ -@var CONFIG_SERVICE=cloud23.cvtt.vpn:6789 -@var ResultPath=/tmp - -{ - "refdata": @inc=http://${CONFIG_SERVICE}/refdata/refdata - , "redis": @inc=http://${CONFIG_SERVICE}/apps/common/redis - , "active_instruments": @inc=http://${CONFIG_SERVICE}/apps/active_instruments - , "cvttcomm": { - "type": "redis" - , "redis_settings": "cvtt-prod-02" - } - , "session_type": "CALENDAR" - , "calendar_session": { - "trading_hours_url": [ - "GET" - , "http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS&start={start}&end={end}" - ] - } - , "books": { - "ALPACA_BK02": {} - , "ALPACA_BK_ILYA_01": {} - } - # -------------------------------------------------------------------- - , "trader": { - "app_channel": "TRDR_03" active_exchanges_: str - - , "is_short_allowed": false - , "exchanges_settings": { - "*": { - "mkt_data": { - "enabled": true - , "streaming": true - }, - } - } - , "algos": @inc=http://${CONFIG_SERVICE}/apps/common/algos - , "trading_pos_diff_pct_trigger": 10.0 - , "trading_depth": { - "USD": 1000.0, - } - } - # -------------------------------------------------------------------- - "executor": { - "app_channel": "EXEC_03" - , "exchanges_settings": { - "COINBASE_AT": { - "order_entry": { - "enabled": true - , "streaming": true - }, - "mkt_data": { - "enabled": false - , "streaming": false - }, - "accounting": { - "enabled": false - , "streaming": false - } - , "credentials_key": { - "read_only": "COINBASE_ADV_RO" - , "trade": "COINBASE_ADV_TRD" - } - } - , "ALPACA_SNDBX": { - "order_entry": { - "enabled": true - , "streaming": true - }, - "mkt_data": { - "enabled": false - , "streaming": false - }, - "accounting": { - "enabled": false - , "streaming": false - } - , "credentials_key": { - "read_only": "ALPACA_SANDBOX" - , "trade": "ALPACA_SANDBOX" - } - } - , "ALPACA_SNDBX-MDPORTAL": { - "order_entry": { - "enabled": true - , "streaming": true - }, - "mkt_data": { - "enabled": false - , "streaming": false - }, - "accounting": { - "enabled": false - , "streaming": false - } - , "credentials_key": { - "read_only": "ALPACA_SANDBOX" - , "trade": "ALPACA_SANDBOX" - } - } - } - } - # -------------------------------------------------------------------- - , "quant": { - "app_channel": "QUANT_03" - , "strategies": @inc=http://${CONFIG_SERVICE}/apps/common/strategies - , "exchanges_settings": { - "*": { - "mkt_data": { - "enabled": true - , "streaming": true - }, - } - # , "COINBASE_AT": { - # "mkt_data": { - # "enabled": true - # , "streaming": true - # }, - # } - } - } - # -------------------------------------------------------------------- - , "risk_manager": { - "app_channel": "RMGR_03" - # , "books": { - # "CVTT_BK01": {} - # } - , "db": { - "positions_table": "prod.cvtt_positions" - , "cred_key": { - "write": "TSDB_TRD_CVTTDATA" - , "read_only": "TSDB_TRD_CVTTDATA" - } - } - , "positions": { - "load_interval_sec": 10.0 - , "to_store_zero_positions": false - , "to_save_unchanged_positions": false - } - , "publish_value_interval": 10.0 - , "exchanges_settings": { - "*": { - "mkt_data": { - "enabled": true - , "streaming": true - }, - "accounting": { - "enabled": true - , "streaming": false - } - , "credentials_key": { - "trade": "ALPACA_SANDBOX" - , "read_only": "ALPACA_SANDBOX" - } - } - } - } - , "trading_recorder": { - # "format": "CSV" - # "format": "TSDB" - "format": "SQLite" - , "csv_settings": { - "order_events_file_path": "%T.order_events.csv" - "position_events_file_path": "%T.position_events.csv" - } - , "tsdb_settings": { - "db": { - "class": "cvttpy.tools.db.timescale.tsdb_client.TimescaleDbClient" - , "cred_key": "TSDB_TRD_CVTTDATA_DB_TEST" - } - , "order_events_table": "backtest.trading_orders" - , "trade_events_table": "backtest.trading_trades" - , "trading_md_summary_table": "backtest.trading_md" - } - , "sqlite_settings": { - # "file_name": "/tmp/%T.cvtt_backtest.db" - "file_name": "/tmp/%T.cvtt_alpaca_sandbox.db" - } - } -} - diff --git a/data/apps/md_rec_monitor.cfg b/data/apps/md_rec_monitor.cfg index 17999fb..072280d 100644 --- a/data/apps/md_rec_monitor.cfg +++ b/data/apps/md_rec_monitor.cfg @@ -22,13 +22,13 @@ "tables": ["bnbfut.md_booktops"] , "exchange_id": "BNBFUT" , "instruments": ["PERP-BTC-USDT"] - , "threshold_sec": 60.0 + , "threshold_sec": 120.0 } , "BNBFUT-2": { "tables": ["bnbfut.md_trades"] , "exchange_id": "BNBFUT" , "instruments": ["PERP-BTC-USDT"] - , "threshold_sec": 30.0 + , "threshold_sec": 120.0 } , "BNBSPOT-1": { "tables": ["bnbspot.md_booktops"] diff --git a/data/apps/md_recorder.cfg b/data/apps/md_recorder.cfg index 811cbc4..9e2902d 100644 --- a/data/apps/md_recorder.cfg +++ b/data/apps/md_recorder.cfg @@ -3,11 +3,15 @@ { "refdata": @inc=http://${CONFIG_SERVICE}/refdata/refdata , "redis": @inc=http://${CONFIG_SERVICE}/apps/common/redis + + // ------------------------------------------------------------------------- + // This is used only for dynamically loaded instrument lists (as for Alpaca) , "active_instruments": @inc=http://${CONFIG_SERVICE}/apps/active_instruments , "cvttcomm": { "type": "redis" - , "redis_settings": "cvtt-prod-02" + , "redis_settings": "cvtt-prod-eqty-01" } + //------------------------------------------------------------------------------ , "md_recorder": { "db": { diff --git a/data/apps/tests/listen_market_data_prod.cfg b/data/apps/tests/listen_market_data_prod.cfg index c63e0ea..4c98156 100644 --- a/data/apps/tests/listen_market_data_prod.cfg +++ b/data/apps/tests/listen_market_data_prod.cfg @@ -13,7 +13,7 @@ } , "cvttcomm": { "type": "redis" - , "redis_settings": "cvtt-prod-02" + , "redis_settings": "cvtt-prod-eqty-01" } , "md_portal": { "app_channel": "MD_PORTAL" diff --git a/data/refdata/exchanges.cfg b/data/refdata/exchanges.cfg index 9a11189..d7d370c 100644 --- a/data/refdata/exchanges.cfg +++ b/data/refdata/exchanges.cfg @@ -419,17 +419,6 @@ , "channel_prefix": "MD_ALPACA_03" } } - , "MD_PORTAL_COINBS_01": { - "exchange_id": "MD_PORTAL" - , "name": "MD Portal (proxy)" - , "type": "PROXY" - - , "mktdata" : { - "class": "cvttpy.exchanges.proxies.md_portal_clnt.mkt_data.MktDataPortalFeed" - , "redis_settings": "cvtt-prod-02" - , "channel_prefix": "MD_COINBS_01" - } - } # ---------------------------- O , "OKX": { "exchange_id": "OKX"