progress

data/admin/cvtt_hosts.cfg

@@ -108,12 +108,6 @@
             "docker": false,
             "supervisor": false,
         },
-        "cvtt-prod-01": {
-            "type": "vm",
-            "users": ["oleg", "cvtt"],
-            "docker": true,
-            "supervisor": false,
-        },
         "cvtt-prod-eqty-01": {
             "type": "vm",
             "users": ["oleg", "cvtt"],
@@ -150,6 +144,18 @@
             "docker": true,
             "supervisor": false,
         },
+        "cvtt-testserver": {
+            "type": "vm",
+            "users": ["oleg", "cvtt"],
+            "docker": true,
+            "supervisor": false,
+        },
+        "cvtt-prod-03": {
+            "type": "vm",
+            "users": ["oleg"],
+            "docker": false,
+            "supervisor": true,
+        },
     },
 
     "cryptovaltrading.com": {

data/admin/git_repositories.cfg

@@ -42,8 +42,8 @@
                 {"homestore": "git@homestore.cvtt.vpn:/opt/git/cvtt2/jupyter_gpu.git" },
             ],
         }
-    },
+    }
-    "allbright": {
+    , "allbright": {
         "allbright-ops": [
             {"origin":	"ssh://allbrigh@allbrightacademy.com:7822/home/allbrigh/git/allbright-ops.git"},
             {"homestore":	"ssh://oleg@homestore.sheynin.home:/opt/git/allbright/allbright-ops.git"},
@@ -53,7 +53,7 @@
             {"homestore":	"ssh://oleg@homestore.sheynin.home:/opt/git/allbright/allbright.git"},
         ]
     }
-    "devel-env": {
+    , "devel-env": {
         "cvtt-dev": [
             {"gitea": "ssh://git@homestore.cvtt.vpn:20022/develop_env/.vscode.cvtt-dev.git"},
         ]
@@ -64,6 +64,15 @@
             {"gitea": "ssh://git@homestore.cvtt.vpn:20022/develop_env/.vscode.oleg-macpro.git"},
         ]
     },
+    , "education-materials": {
+        "Algorithmic Trading Book": [
+            {"origin": "ssh://git@homestore.cvtt.vpn:20022/edu/Algorithmic_Trading_Book.git"}
+        ]
+
+        , "Kubernetes Cookbook": [
+            {"origin": "ssh://git@homestore.cvtt.vpn:20022/edu/Kubernetes_Cookbook.git"}
+        ]
+    }
     "personal": {
         "obsidian_notes": [
             {"origin":	"ssh://cryptova@cryptovaltrading.com:1588/home/cryptova/git/personal/obsidian_notes.git"}

data/apps/active_instruments.cfg

@@ -3,7 +3,7 @@
         "dynamic_stock_list": {
             "exchange_id": "ALPACA"
             , "list_source": "redis"
-            , "redis_settings": "cvtt-prod-02"
+            , "redis_settings": "cvtt-prod-eqty-01"
             , "redis_key": "DAY_STOCK_LIST"
         }
     }

data/apps/algo_trade.cfg

@@ -13,6 +13,45 @@
             , "http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS&start={start}&end={end}"
         ]
     }
+    ###### N E W  S E C T I O N (example)
+    , "simulation": {
+        "speed_factor": 0.0,
+        "time_events_interval_ms": 100
+    }
+    , "market_simulator" : {
+        "market_data": {
+            "sources": [
+                {
+                    "loader_class": "cvttpy.research.time_series.loaders.tsdb_loader.TimescaleTSLoader"
+                    , "credentials_key": "TSDB_MD_CVTTDATA_RO"
+                    # , "interval": "4 hours"
+                    , "interval": "2 hours"
+                    , "from_time": "2024-04-15 09:00:00"
+                }
+                # , {
+                #     "loader_class": "cvttpy.research.time_series.loaders.sqlite_loader.SqliteMdLoader",
+                #     "file_name": "${SourcePath}/20240424.ALPACA.QAT.db"
+                # }
+            ]
+        }
+        , "accounting": {
+            "initial_positions": {
+                "COINBASE": {
+                    "BTC": 0
+                    , "USD": 1000.0
+                }
+            }
+        }
+        , "order_entry": {
+            "execution": {
+                "top_only": true
+                ,"min_order_age_sec": 0.1
+                , "pricing_method": "BY_MKT_TRD_PRICE_BY_ORDER"
+                # , "presubscribe_md_types": ["BOOK_TOP", "TRADES"]
+            }
+        }
+    }
+    ###### N E W  S E C T I O N - end
     , "backtest": {
         "source": {
             "loader_class": "cvttpy.research.time_series.loaders.sqlite_loader.SqliteMdLoader"

data/apps/common/redis.cfg

@@ -14,11 +14,6 @@
         , "read_timeout" : 0.01
         , "ping_timeout_secs": 2.0
     }
-    , "cvtt-prod-02": {
-        "url": "redis://cvtt-prod-02.cvtt.vpn"
-        , "read_timeout" : 0.01
-        , "ping_timeout_secs": 2.0
-    }
     , "cvtt-prod-03": {
         "url": "redis://cvtt-prod-03.cvtt.vpn"
         , "read_timeout" : 0.01

data/apps/cvtt.cfg

@@ -13,6 +13,45 @@
     , "books": {
         "CVTT_BK01": {}
     }
+    ###### N E W  S E C T I O N (example)
+    , "simulation": {
+        "speed_factor": 0.0,
+        "time_events_interval_ms": 100
+    }
+    , "market_simulator" : {
+        "market_data": {
+            "sources": [
+                {
+                    "loader_class": "cvttpy.research.time_series.loaders.tsdb_loader.TimescaleTSLoader"
+                    , "credentials_key": "TSDB_MD_CVTTDATA_RO"
+                    # , "interval": "4 hours"
+                    , "interval": "2 hours"
+                    , "from_time": "2024-04-15 09:00:00"
+                }
+                # , {
+                #     "loader_class": "cvttpy.research.time_series.loaders.sqlite_loader.SqliteMdLoader",
+                #     "file_name": "${SourcePath}/20240424.ALPACA.QAT.db"
+                # }
+            ]
+        }
+        , "accounting": {
+            "initial_positions": {
+                "COINBASE": {
+                    "BTC": 0
+                    , "USD": 1000.0
+                }
+            }
+        }
+        , "order_entry": {
+            "execution": {
+                "top_only": true
+                ,"min_order_age_sec": 0.1
+                , "pricing_method": "BY_MKT_TRD_PRICE_BY_ORDER"
+                # , "presubscribe_md_types": ["BOOK_TOP", "TRADES"]
+            }
+        }
+    }
+    ###### N E W  S E C T I O N - end
     , "backtest": {
         "exchanges_settings": {
             "*": {

data/apps/cvtt_crypto_coinbs.cfg

@@ -13,6 +13,45 @@
     , "books": {
         "CVTT_BK01": {}
     }
+    ###### N E W  S E C T I O N (example)
+    , "simulation": {
+        "speed_factor": 0.0,
+        "time_events_interval_ms": 100
+    }
+    , "market_simulator" : {
+        "market_data": {
+            "sources": [
+                {
+                    "loader_class": "cvttpy.research.time_series.loaders.tsdb_loader.TimescaleTSLoader"
+                    , "credentials_key": "TSDB_MD_CVTTDATA_RO"
+                    # , "interval": "4 hours"
+                    , "interval": "2 hours"
+                    , "from_time": "2024-04-15 09:00:00"
+                }
+                # , {
+                #     "loader_class": "cvttpy.research.time_series.loaders.sqlite_loader.SqliteMdLoader",
+                #     "file_name": "${SourcePath}/20240424.ALPACA.QAT.db"
+                # }
+            ]
+        }
+        , "accounting": {
+            "initial_positions": {
+                "COINBASE": {
+                    "BTC": 0
+                    , "USD": 1000.0
+                }
+            }
+        }
+        , "order_entry": {
+            "execution": {
+                "top_only": true
+                ,"min_order_age_sec": 0.1
+                , "pricing_method": "BY_MKT_TRD_PRICE_BY_ORDER"
+                # , "presubscribe_md_types": ["BOOK_TOP", "TRADES"]
+            }
+        }
+    }
+    ###### N E W  S E C T I O N - end
     , "backtest": {
         "exchanges_settings": {
             "*": {

data/apps/cvtt_eqt_alpaca.cfg

@@ -161,9 +161,8 @@
         }
     }
     , "trading_recorder": {
-        # "format":  "CSV"
+        "app_channel": "TRDREC"
-        # "format":  "TSDB"
+        , "format":  "SQLite" # formats:  "CSV", "TSDB", "SQLite"
-        "format":  "SQLite"
         , "csv_settings": {
             "order_events_file_path": "%T.order_events.csv"
             "position_events_file_path": "%T.position_events.csv"
@@ -179,7 +178,7 @@
         }
         , "sqlite_settings": {
             # "file_name": "/tmp/%T.cvtt_backtest.db"
-            "file_name": "/tmp/%T.cvtt_alpaca_sandbox.db"
+            "file_name": "/logs/%T.cvtt_alpaca_sandbox.db"
         }
     }
 }

data/apps/cvtt_eqt_alpaca_ilya.cfg

@@ -1,180 +0,0 @@
-@var CONFIG_SERVICE=cloud23.cvtt.vpn:6789
-@var ResultPath=/tmp
-
-{
-    "refdata": @inc=http://${CONFIG_SERVICE}/refdata/refdata
-    , "redis": @inc=http://${CONFIG_SERVICE}/apps/common/redis
-    , "active_instruments": @inc=http://${CONFIG_SERVICE}/apps/active_instruments
-    , "cvttcomm": {
-        "type": "redis"
-        , "redis_settings": "cvtt-prod-02"
-    }
-    , "session_type": "CALENDAR"
-    , "calendar_session": {
-        "trading_hours_url": [
-            "GET"
-            , "http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS&start={start}&end={end}"
-        ]
-    }
-    , "books": {
-        "ALPACA_BK02": {}
-        , "ALPACA_BK_ILYA_01": {}
-    }
-    # --------------------------------------------------------------------
-    , "trader": {
-        "app_channel": "TRDR_03"    active_exchanges_: str
-
-        , "is_short_allowed": false
-        , "exchanges_settings": {
-            "*": {
-                "mkt_data": {
-                    "enabled": true
-                    , "streaming": true
-                },
-            }
-        }
-        , "algos": @inc=http://${CONFIG_SERVICE}/apps/common/algos
-        , "trading_pos_diff_pct_trigger": 10.0
-        , "trading_depth": {
-            "USD": 1000.0,
-        }
-    }
-    # --------------------------------------------------------------------
-    "executor": {
-        "app_channel": "EXEC_03"
-        , "exchanges_settings": {
-            "COINBASE_AT": {
-                "order_entry": {
-                    "enabled": true
-                    , "streaming": true
-                },
-                "mkt_data": {
-                    "enabled": false
-                    , "streaming": false
-                },
-                "accounting": {
-                    "enabled": false
-                    , "streaming": false
-                }
-                , "credentials_key": {
-                    "read_only": "COINBASE_ADV_RO"
-                    , "trade": "COINBASE_ADV_TRD"
-                }
-            }
-            , "ALPACA_SNDBX": {
-                "order_entry": {
-                    "enabled": true
-                    , "streaming": true
-                },
-                "mkt_data": {
-                    "enabled": false
-                    , "streaming": false
-                },
-                "accounting": {
-                    "enabled": false
-                    , "streaming": false
-                }
-                , "credentials_key": {
-                    "read_only": "ALPACA_SANDBOX"
-                    , "trade": "ALPACA_SANDBOX"
-                }
-            }
-            , "ALPACA_SNDBX-MDPORTAL": {
-                "order_entry": {
-                    "enabled": true
-                    , "streaming": true
-                },
-                "mkt_data": {
-                    "enabled": false
-                    , "streaming": false
-                },
-                "accounting": {
-                    "enabled": false
-                    , "streaming": false
-                }
-                , "credentials_key": {
-                    "read_only": "ALPACA_SANDBOX"
-                    , "trade": "ALPACA_SANDBOX"
-                }
-            }
-        }
-    }
-    # --------------------------------------------------------------------
-    , "quant": {
-        "app_channel": "QUANT_03"
-        , "strategies": @inc=http://${CONFIG_SERVICE}/apps/common/strategies
-        , "exchanges_settings": {
-            "*": {
-                "mkt_data": {
-                    "enabled": true
-                    , "streaming": true
-                },
-            }
-            # , "COINBASE_AT": {
-            #     "mkt_data": {
-            #         "enabled": true
-            #         , "streaming": true
-            #     },
-            # }
-        }
-    }
-    # --------------------------------------------------------------------
-    , "risk_manager": {
-        "app_channel": "RMGR_03"
-        # , "books": {
-        #     "CVTT_BK01": {}
-        # }
-        , "db": {
-            "positions_table": "prod.cvtt_positions"
-            , "cred_key": {
-                "write": "TSDB_TRD_CVTTDATA"
-                , "read_only": "TSDB_TRD_CVTTDATA"
-            }
-        }
-        , "positions": {
-            "load_interval_sec": 10.0
-            , "to_store_zero_positions": false
-            , "to_save_unchanged_positions": false
-        }
-        , "publish_value_interval": 10.0
-        , "exchanges_settings": {
-            "*": {
-                "mkt_data": {
-                    "enabled": true
-                    , "streaming": true
-                },
-                "accounting": {
-                    "enabled": true
-                    , "streaming": false
-                }
-                , "credentials_key": {
-                    "trade": "ALPACA_SANDBOX"
-                    , "read_only": "ALPACA_SANDBOX"
-                }
-            }
-        }
-    }
-    , "trading_recorder": {
-        # "format":  "CSV"
-        # "format":  "TSDB"
-        "format":  "SQLite"
-        , "csv_settings": {
-            "order_events_file_path": "%T.order_events.csv"
-            "position_events_file_path": "%T.position_events.csv"
-        }
-        , "tsdb_settings": {
-            "db": {
-                "class": "cvttpy.tools.db.timescale.tsdb_client.TimescaleDbClient"
-                , "cred_key": "TSDB_TRD_CVTTDATA_DB_TEST"
-            }
-            , "order_events_table": "backtest.trading_orders"
-            , "trade_events_table": "backtest.trading_trades"
-            , "trading_md_summary_table": "backtest.trading_md"
-        }
-        , "sqlite_settings": {
-            # "file_name": "/tmp/%T.cvtt_backtest.db"
-            "file_name": "/tmp/%T.cvtt_alpaca_sandbox.db"
-        }
-    }
-}
-

data/apps/md_rec_monitor.cfg

@@ -22,13 +22,13 @@
             "tables": ["bnbfut.md_booktops"]
             , "exchange_id": "BNBFUT"
             , "instruments": ["PERP-BTC-USDT"]
-            , "threshold_sec": 60.0
+            , "threshold_sec": 120.0
         }
         , "BNBFUT-2": {
             "tables": ["bnbfut.md_trades"]
             , "exchange_id": "BNBFUT"
             , "instruments": ["PERP-BTC-USDT"]
-            , "threshold_sec": 30.0
+            , "threshold_sec": 120.0
         }
         , "BNBSPOT-1": {
             "tables": ["bnbspot.md_booktops"]

data/apps/md_recorder.cfg

@@ -3,11 +3,15 @@
 {
     "refdata": @inc=http://${CONFIG_SERVICE}/refdata/refdata
     , "redis": @inc=http://${CONFIG_SERVICE}/apps/common/redis
+
+    // -------------------------------------------------------------------------
+    // This is used only for dynamically loaded instrument lists (as for Alpaca)
     , "active_instruments": @inc=http://${CONFIG_SERVICE}/apps/active_instruments
     , "cvttcomm": {
         "type": "redis"
-        , "redis_settings": "cvtt-prod-02"
+        , "redis_settings": "cvtt-prod-eqty-01"
     }
+    //------------------------------------------------------------------------------
 
     , "md_recorder": {
         "db": {

data/apps/tests/listen_market_data_prod.cfg

@@ -13,7 +13,7 @@
     }
     , "cvttcomm": {
         "type": "redis"
-        , "redis_settings": "cvtt-prod-02"
+        , "redis_settings": "cvtt-prod-eqty-01"
     }
     , "md_portal": {
         "app_channel": "MD_PORTAL"

data/refdata/exchanges.cfg

@@ -419,17 +419,6 @@
             , "channel_prefix": "MD_ALPACA_03"
         }
     }
-    , "MD_PORTAL_COINBS_01": {
-        "exchange_id":          "MD_PORTAL"
-        , "name":               "MD Portal (proxy)"
-        , "type":               "PROXY"
-
-        , "mktdata" : {
-            "class":    "cvttpy.exchanges.proxies.md_portal_clnt.mkt_data.MktDataPortalFeed"
-            , "redis_settings": "cvtt-prod-02"
-            , "channel_prefix": "MD_COINBS_01"
-        }
-    }
     # ---------------------------- O
     , "OKX": {
         "exchange_id":          "OKX"