progress 0.0.4

.vscode/launch.json

@@ -31,8 +31,8 @@
                 "PYTHONPATH": "${workspaceFolder}/..",
                 "CONFIG_SERVICE": "cloud16.cvtt.vpn:6789",
                 "MODEL_CONFIG": "vecm",
-                "CVTT_URL": "http://cvtt-tester-01.cvtt.vpn:23456",
-                // "CVTT_URL": "http://dev-server-02.cvtt.vpn:23456",
+                // "CVTT_URL": "http://cvtt-tester-01.cvtt.vpn:23456",
+                "CVTT_URL": "http://dev-server-02.cvtt.vpn:23456",
             },
             "args": [
                 // "--config=${workspaceFolder}/configuration/pair_trader.cfg",

LIBRARIES

@@ -1,2 +0,0 @@
-cvttpy_tools: 1.3.4
-cvttpy_trading: 2.4.1

VERSION

@@ -1 +1 @@
-0.0.2
+0.0.4

apps/pair_trader.py

@@ -141,12 +141,11 @@ class PairTrader(NamedObject):
             )
 
     async def _on_md_summary(self, history: List[MdTradesAggregate], exch_inst: ExchangeInstrument) -> None:
-        # URGENT before calling stragegy, make sure that **BOTH** instruments market data is combined.
         Log.info(f"{self.fname()}: got {exch_inst.details_short()} data")
         self.latest_history_[exch_inst] = history
         if len(self.latest_history_) == 2:
             from itertools import chain
-            all_aggrs = sorted(list(chain.from_iterable(self.latest_history_.values())), key=lambda X: X.time_ns_)
+            all_aggrs = sorted(list(chain.from_iterable(self.latest_history_.values())), key=lambda X: X.aggr_time_ns_)
 
             await self.live_strategy_.on_mkt_data_hist_snapshot(hist_aggr=all_aggrs)
             self.latest_history_ = {}

lib/live/mkt_data_client.py

@@ -163,6 +163,7 @@ class MdSummaryCollector(NamedObject):
             )
             return None
         res = MdSummary.from_REST_response(response=response)
+        Log.info(f"DEBUG *** {self.exch_inst_.base_asset_id_}: {res[-1].tstamp_}")
         return None if len(res) == 0 else res[-1]
 
     def is_empty(self) -> bool:
@@ -195,15 +196,16 @@ class MdSummaryCollector(NamedObject):
         Log.info(f"{self.fname()} Timer for {self.exch_inst_.details_short()} is set to run in {start_in} sec")
 
     def next_load_time(self) -> NanosT:
+        ALLOW_LAG_SEC = 1
         curr_sec = int(current_seconds())
-        return (curr_sec - curr_sec % self.interval_sec_) + self.interval_sec_ + 2
+        return (curr_sec - curr_sec % self.interval_sec_) + self.interval_sec_ + ALLOW_LAG_SEC
 
     async def _load_new(self) -> None:
 
         last: Optional[MdSummary] = self.get_last()
         if not last:
             Log.warning(f"{self.fname()}: did not get last update")
-        elif not self.is_empty() and last.ts_ns_ <= self.history_[-1].time_ns_:
+        elif not self.is_empty() and last.ts_ns_ <= self.history_[-1].aggr_time_ns_:
             Log.info(
                 f"{self.fname()}: Received {last}. Already Have: {self.history_[-1]}"
             )

lib/live/rest_client.py

@@ -1,18 +1,12 @@
 from __future__ import annotations
 
-import asyncio
-from typing import Callable, Dict, Any, List, Optional
+from typing import Dict
 import time
 
 import requests
 
 from cvttpy_tools.base import NamedObject
-from cvttpy_tools.logger import Log
-from cvttpy_tools.config import Config
-from cvttpy_tools.timer import Timer
 
-from cvttpy_tools.timeutils import NanoPerSec, NanosT, current_nanoseconds, current_seconds
-from cvttpy_trading.trading.mkt_data.historical_md import HistMdBar
 
 
 class RESTSender(NamedObject):

lib/pt_strategy/live/live_strategy.py

@@ -9,7 +9,7 @@ from cvttpy_tools.base import NamedObject
 from cvttpy_tools.app import App
 from cvttpy_tools.config import Config
 from cvttpy_tools.settings.cvtt_types import IntervalSecT
-from cvttpy_tools.timeutils import SecPerHour, current_nanoseconds, NanoPerSec
+from cvttpy_tools.timeutils import NanosT, SecPerHour, current_nanoseconds, NanoPerSec, format_nanos_utc
 from cvttpy_tools.logger import Log
 
 # ---
@@ -42,14 +42,14 @@ class PtLiveStrategy(NamedObject):
     # for presentation: history of prediction values and trading signals
     predictions_df_: pd.DataFrame
     trading_signals_df_: pd.DataFrame
+    allowed_md_lag_sec_: int
+
 
     def __init__(
         self,
         config: Config,
         pairs_trader: PairTrader,
     ):
-        # import copy
-        # self.config_ = Config(json_src=copy.deepcopy(config.data()))
         self.config_ = config
 
         self.pairs_trader_ = pairs_trader
@@ -83,6 +83,8 @@ class PtLiveStrategy(NamedObject):
         )
         assert self.history_depth_sec_ > 0, "history_depth_hours cannot be 0"
 
+        self.allowed_md_lag_sec_ = self.config_.get_value("allowed_md_lag_sec", 3)
+
         await self.pairs_trader_.subscribe_md()
 
         self.open_threshold_ = self.config_.get_value(
@@ -132,17 +134,33 @@ class PtLiveStrategy(NamedObject):
             await self._send_trading_instructions(trading_instructions)
 
     def _is_md_actual(self, hist_aggr: List[MdTradesAggregate]) -> bool:
-        curr_ns = current_nanoseconds()
-        LAG_THRESHOLD = 5 * NanoPerSec
-        
         if len(hist_aggr) == 0:
             Log.warning(f"{self.fname()} list of aggregates IS EMPTY")
             return False
+
+        curr_ns = current_nanoseconds()
+        
         # MAYBE check market data length
-        lag_ns = curr_ns - hist_aggr[-1].time_ns_
-        if lag_ns > LAG_THRESHOLD:
-            Log.warning(f"{self.fname()} {hist_aggr[-1].exch_inst_.details_short()} Lagging {int(lag_ns/NanoPerSec)} seconds")
+        
+        # at 18:05:01 we should see data for 18:04:00
+        lag_sec = (curr_ns - hist_aggr[-1].aggr_time_ns_) / NanoPerSec - self.interval_sec()
+        if lag_sec > self.allowed_md_lag_sec_:
+            Log.warning(
+                f"{self.fname()} {hist_aggr[-1].exch_inst_.details_short()}"
+                f" Lagging {int(lag_sec)} > {self.allowed_md_lag_sec_} seconds:"
+                f"\n{len(hist_aggr)} records"
+                f"\n{hist_aggr[-1].exch_inst_.base_asset_id_}: {hist_aggr[-1].tstamp()}"
+                f"\n{hist_aggr[-2].exch_inst_.base_asset_id_}: {hist_aggr[-2].tstamp()}"
+            )
             return False
+        else:
+            Log.info(
+                f"{self.fname()} {hist_aggr[-1].exch_inst_.details_short()}"
+                f" Lag {int(lag_sec)} <= {self.allowed_md_lag_sec_} seconds"
+                f"\n{len(hist_aggr)} records"
+                f"\n{hist_aggr[-1].exch_inst_.base_asset_id_}: {hist_aggr[-1].tstamp()}"
+                f"\n{hist_aggr[-2].exch_inst_.base_asset_id_}: {hist_aggr[-2].tstamp()}"
+            )
         return True
 
     def _create_md_df(self, hist_aggr: List[MdTradesAggregate]) -> pd.DataFrame:
@@ -163,8 +181,8 @@ class PtLiveStrategy(NamedObject):
             rows.append(
                 {
                     # convert nanoseconds → tz-aware pandas timestamp
-                    "tstamp": pd.to_datetime(aggr.time_ns_, unit="ns", utc=True),
-                    "time_ns": aggr.time_ns_,
+                    "tstamp": pd.to_datetime(aggr.aggr_time_ns_, unit="ns", utc=True),
+                    "time_ns": aggr.aggr_time_ns_,
                     "symbol": exch_inst.instrument_id().split("-", 1)[1],
                     "exchange_id": exch_inst.exchange_id_,
                     "instrument_id": exch_inst.instrument_id(),