progress

src/tools/data_loader.py

@@ -1,5 +1,5 @@
 import sqlite3
-from typing import Dict
+from typing import Dict, List, cast
 import pandas as pd
 
 
@@ -83,9 +83,53 @@ def load_market_data(datafile: str, config: Dict) -> pd.DataFrame:
     df = df[(df["tstamp"] >= start_time) & (df["tstamp"] <= end_time)]
     df["tstamp"] = pd.to_datetime(df["tstamp"])
 
-    return df
+    return cast(pd.DataFrame, df)
 
 
+def get_available_instruments_from_db(datafile: str, config: Dict) -> List[str]:
+    """
+    Auto-detect available instruments from the database by querying distinct instrument_id values.
+    Returns instruments without the configured prefix.
+    """
+    try:
+        conn = sqlite3.connect(datafile)
+
+        # Build exclusion list with full instrument_ids
+        exclude_instruments = config.get("exclude_instruments", [])
+        prefix = config.get("instrument_id_pfx", "")
+        exclude_instrument_ids = [f"{prefix}{inst}" for inst in exclude_instruments]
+        
+        # Query to get distinct instrument_ids
+        query = f"""
+        SELECT DISTINCT instrument_id 
+        FROM {config['db_table_name']} 
+        WHERE exchange_id = ?
+        """
+        
+        # Add exclusion clause if there are instruments to exclude
+        if exclude_instrument_ids:
+            placeholders = ','.join(['?' for _ in exclude_instrument_ids])
+            query += f" AND instrument_id NOT IN ({placeholders})"
+            cursor = conn.execute(query, (config["exchange_id"],) + tuple(exclude_instrument_ids))
+        else:
+            cursor = conn.execute(query, (config["exchange_id"],))
+        instrument_ids = [row[0] for row in cursor.fetchall()]
+        conn.close()
+
+        # Remove the configured prefix to get instrument symbols
+        instruments = []
+        for instrument_id in instrument_ids:
+            if instrument_id.startswith(prefix):
+                symbol = instrument_id[len(prefix) :]
+                instruments.append(symbol)
+            else:
+                instruments.append(instrument_id)
+
+        return sorted(instruments)
+
+    except Exception as e:
+        print(f"Error auto-detecting instruments from {datafile}: {str(e)}")
+        return []
 
 
 # if __name__ == "__main__":