"""
Feature Engineering Component.

Handles adding base features (cyclical, imbalance, TA) and selecting features
using Logistic Regression (L1) and Variance Inflation Factor (VIF).
"""

import pandas as pd
import numpy as np
import logging
import json
import scipy.stats as st # Added for z-score
from typing import List
import time
import pandas_ta as ta # <<< ADDED
import warnings # <<< ADDED for catching warnings

from sklearn.linear_model import LogisticRegression, LinearRegression # Added LinearRegression
from sklearn.feature_selection import SelectFromModel
from statsmodels.stats.outliers_influence import variance_inflation_factor
import statsmodels.api as sm
# Added RollingOLS for trend-fit residuals
from statsmodels.regression.rolling import RollingOLS 
from sklearn.preprocessing import PolynomialFeatures # <<< ADDED for interactions

# Import TA library functions directly
from ta.volatility import AverageTrueRange, KeltnerChannel, BollingerBands # Added BollingerBands
from ta.momentum import RSIIndicator
from ta.trend import EMAIndicator, MACD
# Remove specific ta.volume imports if using pandas_ta alias

logger = logging.getLogger(__name__)

_EPS = 1e-6

# --- Helper function for safe division ---
def safe_divide(numerator, denominator, default=0.0):
    """Performs division, returning default value where denominator is near zero or NaN."""
    mask = (np.abs(denominator) < _EPS) | np.isnan(denominator) | np.isnan(numerator)
    result = np.divide(numerator, denominator, out=np.full_like(numerator, default, dtype=np.float64), where=~mask)
    result[mask] = default
    return result

# --- Helper function for rolling regression residuals ---
def rolling_regression_residual(y, window=3):
    """Calculates the residual of a rolling OLS regression of y against time."""
    if len(y) < window:
        return pd.Series(np.nan, index=y.index)
    
    try:
        # Ensure y is numeric
        y_numeric = pd.to_numeric(y, errors='coerce')
        
        # Prepare Exog Data (Add constant to x)
        x_trend = np.arange(len(y))
        X_exog = sm.add_constant(x_trend, prepend=True)
        X_exog_df = pd.DataFrame(X_exog, index=y.index, columns=['const', 'time'])
        
        # Align y and X
        y_numeric, X_aligned = y_numeric.align(X_exog_df, join='inner', axis=0)
        
        if y_numeric.isnull().all() or X_aligned.isnull().all().all():
             logger.warning("Input to RollingOLS contains all NaNs after alignment.")
             return pd.Series(np.nan, index=y.index)
             
        rols = RollingOLS(endog=y_numeric, exog=X_aligned, window=window, min_nobs=window)
        
        # <<< Catch RuntimeWarning during fit >>>
        with warnings.catch_warnings():
            # Ignore the specific warning about log(ssr) when ssr is zero
            warnings.filterwarnings(
                'ignore', 
                message='divide by zero encountered in log',
                category=RuntimeWarning
            )
            results = rols.fit()
        # <<< End Catch >>>
        
        # --- Re-calculate Fitted Values & Residuals (Fix for AttributeError) --- # 
        params = results.params
        fitted_values = (X_aligned * params).sum(axis=1)
        residuals = y_numeric - fitted_values # Correct calculation
        # --- End Re-calculation --- #
        
        # Reindex residuals to match the original input index, filling gaps with NaN
        return residuals.reindex(y.index)

    except Exception as e:
        logger.error(f"Error during rolling regression: {e}", exc_info=True)
        return pd.Series(np.nan, index=y.index)

class FeatureEngineer:
    """Encapsulates feature creation and selection logic."""

    def __init__(self, config: dict): # Changed to accept config
        """
        Initialize the FeatureEngineer.

        Args:
            config (dict): Pipeline configuration dictionary.
        """
        self.config = config
        # Minimal whitelist definition can be moved here or kept separate
        # For now, assuming it's defined elsewhere or passed via config if needed
        self.minimal_whitelist = config.get('features', {}).get('minimal_whitelist', [])
        if not self.minimal_whitelist:
             logger.warning("Minimal whitelist not found in config or is empty.")
             # Define a default fallback if necessary
             self.minimal_whitelist = [
                 "return_1m", "return_15m", "return_60m", "ATR_14", "volatility_14d",
                 "chaikin_AD_10", "svi_10", "EMA_10", "EMA_50",
                 "hour_sin", "hour_cos",
             ]
        logger.info(f"FeatureEngineer initialized. Minimal whitelist: {self.minimal_whitelist}")

    def _add_cyclical_features(self, df: pd.DataFrame) -> pd.DataFrame:
        """Adds sine and cosine transformations of the hour and week progress."""
        if not isinstance(df.index, pd.DatetimeIndex):
            logger.warning("Index is not DatetimeIndex. Skipping cyclical features.")
            # Add placeholders if needed
            df['hour_sin'] = 0.0
            df['hour_cos'] = 1.0 
            df['week_sin'] = 0.0
            df['week_cos'] = 1.0 
            return df
            
        timestamp_source = df.index
        logger.info("Adding cyclical hour features (sin/cos)...")
        # --- Hourly Features --- #
        hours_in_day = 24
        df['hour_sin'] = np.sin(2 * np.pi * timestamp_source.hour / hours_in_day)
        df['hour_cos'] = np.cos(2 * np.pi * timestamp_source.hour / hours_in_day)

        # --- Weekly Features (Task 2.2) --- #
        logger.info("Adding cyclical weekly features (sin/cos)...")
        # Calculate time elapsed in minutes within the week (0=Monday 00:00, max=7*24*60)
        # dayofweek: Monday=0, Sunday=6
        minutes_in_week = 7 * 24 * 60
        time_in_week_minutes = (timestamp_source.dayofweek * 24 * 60 + 
                                timestamp_source.hour * 60 + 
                                timestamp_source.minute)
        
        df['week_sin'] = np.sin(2 * np.pi * time_in_week_minutes / minutes_in_week)
        df['week_cos'] = np.cos(2 * np.pi * time_in_week_minutes / minutes_in_week)
        # --- End Weekly Features --- #

        return df

    def _add_imbalance_features(self, df: pd.DataFrame) -> pd.DataFrame:
        """Add Chaikin AD line, signed volume imbalance, gap imbalance, PVT, MFI."""
        logger.info("Adding imbalance features...")
        if not {"open", "high", "low", "close", "volume"}.issubset(df.columns):
            logger.warning("Missing required columns for imbalance features. Skipping.")
            return df

        df_shifted = df.shift(1) # Use shifted data for calculations
        df_results = pd.DataFrame(index=df.index) # Store new features here
        # Initialize all expected columns with NaN to ensure float dtype from the start
        cols_to_init = [
            "chaikin_AD_10", "svi_10", "signed_vol_norm_std_10", 
            "gap_imbalance", "pvt", "mfi_14"
        ]
        for col in cols_to_init:
            # Explicitly initialize with float64 dtype
            df_results[col] = pd.Series(np.nan, index=df.index, dtype=np.float64)

        try:
            # Chaikin AD (uses non-shifted internally via clv)
            clv = safe_divide((df["close"] - df["low"]) - (df["high"] - df["close"]),
                              (df["high"] - df["low"]), default=0.0)
            df_results["chaikin_AD_10"] = (clv * df["volume"]).rolling(10).sum()

            # Signed Volume Imbalance (Rolling Sum)
            signed_vol = np.where(df_shifted["close"] >= df_shifted["open"], df_shifted["volume"], -df_shifted["volume"])
            df_results["svi_10"] = pd.Series(signed_vol, index=df.index).rolling(10).sum()

            # Signed Volume / Rolling Vol Std (window=10)
            vol_std_10 = df_shifted['volume'].rolling(window=10, min_periods=5).std()
            df_results["signed_vol_norm_std_10"] = safe_divide(pd.Series(signed_vol, index=df.index), vol_std_10, default=0.0)

            # Gap Imbalance
            med_vol = df_shifted["volume"].rolling(50).median()
            gap_up = (df_shifted["low"] > df["high"].shift(2)) & (df_shifted["volume"] > 2 * med_vol) # Check against t-2 high
            gap_dn = (df_shifted["high"] < df["low"].shift(2)) & (df_shifted["volume"] > 2 * med_vol) # Check against t-2 low
            df_results["gap_imbalance"] = gap_up.astype(int) - gap_dn.astype(int)
            
            # Price-Volume Trend (PVT) using ta library
            # Remove individual np.nan initialization here
            try:
                 # Use pandas_ta function call
                 df_results["pvt"] = ta.pvt(close=df_shifted["close"], volume=df_shifted["volume"])
            except Exception as pvt_e:
                 logger.error(f"Error calculating PVT: {pvt_e}")
                 df_results["pvt"] = 0.0
                 
            # Money Flow Index (MFI) using ta library (default window 14)
            # Remove individual np.nan initialization here
            try:
                 # Use pandas_ta function call
                 # REMOVE .astype(float) here as the target column dtype is already set
                 df_results["mfi_14"] = ta.mfi(high=df_shifted["high"], low=df_shifted["low"], close=df_shifted["close"], volume=df_shifted["volume"], window=14).astype(float)
            except Exception as mfi_e:
                 logger.error(f"Error calculating MFI: {mfi_e}")
                 df_results["mfi_14"] = 50.0 # Ensure fallback is float

            # Fill NaNs introduced within this method
            imbalance_cols = ["chaikin_AD_10", "svi_10", "signed_vol_norm_std_10", "gap_imbalance", "pvt", "mfi_14"]
            for col in imbalance_cols:
                 if col in df_results.columns:
                      # Replace inplace fillna
                      df_results[col] = df_results[col].ffill()
                      df_results[col] = df_results[col].bfill()
                      df_results[col] = df_results[col].fillna(0) # Final fill for start/end

            # Add results to original DataFrame
            for col in imbalance_cols:
                 if col in df_results.columns:
                     # Explicitly cast to float to avoid dtype warnings
                     df[col] = df_results[col].astype(float) 
                     
            logger.info("Successfully added imbalance features (incl. PVT, MFI).")
        except Exception as e:
            logger.error(f"Error calculating imbalance features: {e}", exc_info=True)

        return df

    def _add_vol_norm_returns(self, df: pd.DataFrame) -> pd.DataFrame:
        """Adds volatility-normalized returns."""
        logger.info("Adding volatility-normalized returns...")
        # Expects return_1m, return_15m, return_60m to exist from _add_ta_features
        return_cols = [col for col in df.columns if col.startswith('return_') and col.endswith('m')]
        if not return_cols:
            logger.warning("No base return columns (e.g., return_1m) found. Skipping volatility-normalized returns.")
            return df

        window = 10 # Define the rolling window for std dev
        logger.info(f"Using window={window} for volatility normalization standard deviation.")

        for col in return_cols:
            norm_col_name = col.replace("return_", "return_norm_")
            try:
                rolling_std = df[col].rolling(window=window, min_periods=window // 2).std()
                # Handle potential division by zero or NaNs
                df[norm_col_name] = df[col] / (rolling_std + _EPS)
                # Fill any resulting NaNs/Infs (e.g., from initial periods or zero std dev)
                # Fix inplace replace warning
                df[norm_col_name] = df[norm_col_name].replace([np.inf, -np.inf], np.nan)
                # Replace inplace fillna
                df[norm_col_name] = df[norm_col_name].ffill()
                df[norm_col_name] = df[norm_col_name].bfill()
                df[norm_col_name] = df[norm_col_name].fillna(0) # Final fill
                logger.debug(f"Added {norm_col_name}")
            except Exception as e:
                logger.error(f"Error calculating {norm_col_name}: {e}")
                df[norm_col_name] = 0.0 # Add placeholder on error

        logger.info("Successfully added volatility-normalized returns.")
        return df

    def _add_ta_features(self, df: pd.DataFrame) -> pd.DataFrame:
        """Adds TA features using the 'ta' library and config parameters."""
        logger.info("Adding TA features...")
        required_cols = {'open', 'high', 'low', 'close', 'volume'}
        if not required_cols.issubset(df.columns):
             logger.warning(f"Missing required columns for TA features ({required_cols - set(df.columns)}). Skipping TA.")
             return df

        # Get feature windows from config
        feat_cfg = self.config.get('features', {})
        atr_window = feat_cfg.get('atr_window', 14)
        rsi_window = feat_cfg.get('rsi_window', 14)
        macd_fast = feat_cfg.get('macd_fast', 12)
        macd_slow = feat_cfg.get('macd_slow', 26)
        macd_signal = feat_cfg.get('macd_signal', 9)
        ema_10_window = 10 # Hardcoded for now, could be configurable
        ema_50_window = 50 # Hardcoded for now, could be configurable
        volatility_window_days = 14 # Hardcoded for now, could be configurable

        # Apply shift(1) to prevent lookahead bias in TA features based on close
        # Features will be calculated based on data up to t-1
        df_shifted = df.shift(1)
        df_ta = pd.DataFrame(index=df.index) # Create empty DF to store results aligned with original index

        try:
            # Calculate returns first (use shifted close)
            # Fill NaNs robustly before pct_change on the *shifted* data
            close_filled = df_shifted["close"].bfill().ffill()
            df_ta["return_1m"] = close_filled.pct_change()
            df_ta["return_15m"] = close_filled.pct_change(15)
            df_ta["return_60m"] = close_filled.pct_change(60)

            # Calculate TA features using ta library on *shifted* data
            df_ta[f"ATR_{atr_window}"] = AverageTrueRange(df_shifted['high'], df_shifted['low'], df_shifted['close'], window=atr_window).average_true_range()

            # Daily volatility (use calculated 1m return)
            # Assumes 1-min bars; adjust multiplier if interval changes
            vol_roll_window = 60 * 24 * volatility_window_days
            df_ta[f"volatility_{volatility_window_days}d"] = safe_divide(1, 1) * df_ta["return_1m"].rolling(vol_roll_window, min_periods=vol_roll_window // 2).std() # Corrected line

            # EMA 10 / 50 + MACD using ta library (on shifted close)
            df_ta[f"EMA_{ema_10_window}"] = EMAIndicator(df_shifted["close"], ema_10_window).ema_indicator()
            df_ta[f"EMA_{ema_50_window}"] = EMAIndicator(df_shifted["close"], ema_50_window).ema_indicator()
            macd = MACD(df_shifted["close"], window_slow=macd_slow, window_fast=macd_fast, window_sign=macd_signal)
            df_ta["MACD"] = macd.macd()
            df_ta["MACD_signal"] = macd.macd_signal()

            # RSI using ta library (on shifted close)
            df_ta[f"RSI_{rsi_window}"] = RSIIndicator(df_shifted["close"], window=rsi_window).rsi()

            # Handle potential NaNs introduced by TA calculations
            df_ta.bfill(inplace=True)
            df_ta.ffill(inplace=True)

            # Add the calculated TA features back to the original df
            ta_cols_to_add = [col for col in df_ta.columns if col not in df.columns]
            for col in ta_cols_to_add:
                df[col] = df_ta[col]
            logger.info("Successfully added TA features.")

        except Exception as e:
            logger.error(f"Error calculating TA features: {e}", exc_info=True)

        return df

    def _add_microstructure_features(self, df: pd.DataFrame) -> pd.DataFrame:
        """Adds bar-level microstructure features based on revisions.txt 1-B."""
        logger.info("Adding microstructure features...")
        required_cols = {"open", "high", "low", "close", "volume"}
        if not required_cols.issubset(df.columns):
            logger.warning("Missing required columns for microstructure features. Skipping.")
            return df
            
        df_micro = pd.DataFrame(index=df.index) # Create empty DF for results
        df_shifted = df.shift(1) # Shift all inputs to avoid lookahead
        
        try:
            # --- Revision 2: Add guards for potentially all-NaN columns --- #
            # Check if high/low are all NaNs before using them
            high_valid = not df_shifted["high"].isnull().all()
            low_valid = not df_shifted["low"].isnull().all()
            # --- End Revision 2 --- #

            # 1. Spread proxy
            # Calculation: (high/low - 1)
            # --- Revision 2 Guard --- #
            if high_valid and low_valid:
                df_micro["spread_proxy"] = df_shifted["high"] / (df_shifted["low"] + _EPS) - 1
            else:
                logger.warning("Spread proxy cannot be calculated (high/low contains all NaNs). Filling with 0.")
                df_micro["spread_proxy"] = 0.0
            # --- End Revision 2 --- #
            
            # 2. Vol-norm volume spike (using 1 day = 1440 min rolling mean/std)
            # Calculation: (volume - rolling_mean_1440) / rolling_std_1440
            # Use min_periods for robustness at the start
            vol_roll_mean = df_shifted["volume"].rolling(window=1440, min_periods=int(1440*0.5)).mean()
            vol_roll_std = df_shifted["volume"].rolling(window=1440, min_periods=int(1440*0.5)).std()
            df_micro["vol_norm_volume_spike"] = (df_shifted["volume"] - vol_roll_mean) / (vol_roll_std + _EPS)

            # 3. Return asymmetry
            # Calculation: (close - open) / (high - low)
            intrabar_range = df_shifted["high"] - df_shifted["low"] + _EPS
            df_micro["return_asymmetry"] = (df_shifted["close"] - df_shifted["open"]) / intrabar_range
            
            # 4. Close-location value
            # Calculation: abs(close - (high + low) / 2) / (high - low)
            # --- Revision 2 Guard --- #
            if high_valid and low_valid:
                mid_point = (df_shifted["high"] + df_shifted["low"]) / 2
                # Ensure intrabar_range is recalculated here if needed, or use the one from above
                intrabar_range_clv = df_shifted["high"] - df_shifted["low"] + _EPS 
                df_micro["close_location_value"] = abs(df_shifted["close"] - mid_point) / intrabar_range_clv
            else:
                logger.warning("Close-location value cannot be calculated (high/low contains all NaNs). Filling with 0.")
                df_micro["close_location_value"] = 0.0
            # --- End Revision 2 --- #
            
            # 5. Keltner band position (using EMA 20 and ATR 20)
            # Calculation: (close - ema20) / (atr20 * 2) 
            # Need to calculate EMA(20) and ATR(20) if not already present from _add_ta_features
            if 'EMA_20' not in df_shifted.columns:
                 # Use ta lib directly on shifted data 
                 ema_20_calc = EMAIndicator(df_shifted["close"], window=20).ema_indicator()
            else:
                 ema_20_calc = df_shifted['EMA_20']
                 
            if 'ATR_20' not in df_shifted.columns: 
                 # Use ta lib directly on shifted data
                 atr_20_calc = AverageTrueRange(df_shifted['high'], df_shifted['low'], df_shifted['close'], window=20).average_true_range()
            else:
                 atr_20_calc = df_shifted['ATR_20']
                 
            # Keltner calculation using pre-calculated or just-calculated indicators
            keltner_center = ema_20_calc 
            keltner_deviation = (atr_20_calc * 2) # As per formula spec (atr20 * 2)
            df_micro["keltner_band_pos"] = (df_shifted["close"] - keltner_center) / (keltner_deviation + _EPS)

            # Fill NaNs introduced by rolling calculations or division by zero
            micro_cols = df_micro.columns.tolist()
            # Simple fillna(0) might be problematic. Let's try bfill/ffill first.
            df_micro.bfill(inplace=True)
            df_micro.ffill(inplace=True)
            # Final fill with 0 if any NaNs remain (e.g., at the very start)
            df_micro.fillna(0, inplace=True)
            
            # Add calculated features back to original df
            for col in micro_cols:
                 if col not in df.columns:
                      df[col] = df_micro[col]
                 else: # Overwrite if column already exists (e.g., from previous runs/versions) 
                      df[col] = df_micro[col]
                      logger.warning(f"Overwriting existing column '{col}' with new microstructure feature.")
            logger.info(f"Successfully added microstructure features: {micro_cols}")
            
        except ImportError:
             logger.error("'ta' library not found. Cannot calculate Keltner/EMA/ATR needed for microstructure features. Skipping.")
        except Exception as e:
            logger.error(f"Error calculating microstructure features: {e}", exc_info=True)
            
        return df

    def add_base_features(self, df: pd.DataFrame) -> pd.DataFrame:
        """
        Adds a standard set of base features: cyclical, imbalance, TA, microstructure,
        and additional features from the grab-bag list.

        Args:
            df (pd.DataFrame): Input DataFrame with OHLCV data and DatetimeIndex.

        Returns:
            pd.DataFrame: DataFrame with added features.
        """
        start_time = time.time()
        logger.info(f"Starting base feature engineering on {df.shape[0]} rows...")
        # Ensure index is datetime
        if not isinstance(df.index, pd.DatetimeIndex):
            try:
                df.index = pd.to_datetime(df.index)
                logger.info("Converted index to DatetimeIndex.")
            except Exception as e:
                logger.error(f"Failed to convert index to DatetimeIndex: {e}. Some features may fail.")
                # Depending on strictness, could return df here or raise error

        # --- Apply Feature Steps --- #
        # Order is somewhat important based on dependencies
        df = self._add_ta_features(df) # Needs OHLCV
        df = self._add_vol_norm_returns(df) # Needs returns from _add_ta_features
        df = self._add_cyclical_features(df) # Needs index
        df = self._add_imbalance_features(df) # Needs OHLCV, Volume; Adds PVT, MFI
        df = self._add_microstructure_features(df) # Needs OHLCV, Volume, potentially EMA/ATR from _add_ta_features
        
        # <<< Call NEW feature methods >>>
        df = self._add_candle_features(df) # Needs OHLC
        df = self._add_volatility_channel_features(df) # Needs Close, EMA/ATR
        df = self._add_momentum_trend_features(df) # Needs Close, return_1m
        df = self._add_extra_seasonality_features(df) # Needs index
        # <<< End CALL NEW >>>

        # <<< CALL Interaction Method HERE
        df = self._add_interaction_features(df)

        # --- Handle bar_imputed if present --- #
        if 'bar_imputed' in df.columns:
            # Ensure it's integer type and fill potential NaNs from joins/shifts
            df['bar_imputed'] = df['bar_imputed'].fillna(0).astype(int)
        else:
             # If missing after all feature steps, add it as 0
             logger.warning("'bar_imputed' column not found after feature engineering. Adding column filled with 0.")
             df['bar_imputed'] = 0
        # --- End Handle bar_imputed --- #

        # --- Final Check & Clean --- # 
        # Convert boolean columns potentially created by TA-Lib to int/float
        for col in df.select_dtypes(include='bool').columns:
            logger.debug(f"Converting boolean column '{col}' to int.")
            df[col] = df[col].astype(int)
            
        # Replace any remaining infinities just in case
        df.replace([np.inf, -np.inf], np.nan, inplace=True)
        
        # Optional: Final check for NaNs - uncomment if needed for debugging
        # nan_counts = df.isnull().sum()
        # if nan_counts.sum() > 0:
        #     logger.warning(f"NaNs remain after feature engineering:")
        #     logger.warning(nan_counts[nan_counts > 0])
        #     # Decide on final fill strategy - maybe ffill/bfill again or fillna(0)
        #     df.ffill(inplace=True)
        #     df.bfill(inplace=True)
        #     df.fillna(0, inplace=True)
        # --- End Final Check --- #

        end_time = time.time()
        logger.info(f"Base feature engineering complete. Shape: {df.shape}. Time: {end_time - start_time:.2f}s")
        return df

    def select_features(self, X_train_raw: pd.DataFrame, y_dir_train: pd.Series) -> list:
        """
        Performs feature selection using:
        1. Logistic Regression (L1 penalty) to find predictive features.
        2. Variance Inflation Factor (VIF) to remove multi-collinear features.
        Uses parameters from the 'features' section of the config.

        Args:
            X_train_raw (pd.DataFrame): Raw training features (unscaled).
            y_dir_train (pd.Series): Training direction labels (ordinal).

        Returns:
            list: The final list of selected feature names.
        """
        logger.info("Starting feature selection process...")
        if X_train_raw.empty:
            logger.error("Input DataFrame X_train_raw is empty. Skipping feature selection.")
            return []

        # Get parameters from config
        feat_cfg = self.config.get('features', {})
        vif_threshold = feat_cfg.get('vif_threshold', 10.0) # Default VIF threshold
        # Increase C for less penalty (C is the inverse of regularization strength)
        logreg_c = feat_cfg.get('logreg_c', 1.0) # Default L1 regularization strength (Higher C = less penalty)
        min_features = feat_cfg.get('min_features_after_selection', 5) # Default min features
        selection_method = feat_cfg.get('selection_method', 'logreg_vif') # Default method

        # Ensure targets are not in feature set
        X_train_only_features = X_train_raw.select_dtypes(include=np.number)
        if y_dir_train.name in X_train_only_features.columns:
            X_train_only_features = X_train_only_features.drop(columns=[y_dir_train.name])
            logger.debug(f"Removed target column '{y_dir_train.name}' from features before selection.")
            
        # Handle potential NaNs/Infs (important before LogReg/VIF)
        X_train_clean = X_train_only_features.replace([np.inf, -np.inf], np.nan).fillna(0)
        if X_train_clean.isnull().any().any():
            logger.warning("NaNs still present after fillna(0). Feature selection might be unstable.")

        if X_train_clean.empty:
             logger.error("Feature set is empty after cleaning. Skipping selection.")
             return []

        # --- 1. Univariate Filtering (New Step) --- #
        univariate_quantile_threshold = feat_cfg.get('univariate_quantile_threshold', 0.30) # Keep top 70% by default
        logger.info(f"Performing univariate filtering (keeping features above {univariate_quantile_threshold} quantile of correlation with target)...")
        try:
            # Ensure y_dir_train is aligned with X_train_clean and numeric
            y_aligned, X_aligned = y_dir_train.align(X_train_clean, join='inner', axis=0)
            y_numeric = pd.to_numeric(y_aligned, errors='coerce').fillna(0) # Convert to numeric, fill NaNs
            X_train_clean = X_aligned # Use aligned X from now on

            if X_train_clean.empty:
                 logger.error("Feature set empty after aligning with target for univariate filter. Skipping selection.")
                 return []

            corrs = X_train_clean.apply(lambda col: abs(np.corrcoef(col, y_numeric)[0, 1]) if col.var() > _EPS else 0.0)
            corrs.fillna(0, inplace=True) # Fill NaN correlations (e.g., from zero-variance columns)
            
            if corrs.empty or corrs.isnull().all():
                 logger.warning("Could not compute valid correlations for univariate filter. Skipping this step.")
                 X_train_univariate_filtered = X_train_clean # Pass all cleaned features to next step
            else:
                 quantile_value = corrs.quantile(univariate_quantile_threshold)
                 keep_mask = corrs >= quantile_value
                 univariate_selected_features = corrs[keep_mask].index.tolist()
                 if not univariate_selected_features:
                     logger.warning(f"Univariate filter removed all features (threshold {quantile_value:.4f}). Keeping all features to proceed.")
                     X_train_univariate_filtered = X_train_clean
                 else:
                     X_train_univariate_filtered = X_train_clean[univariate_selected_features]
                     logger.info(f"Univariate filter kept {len(univariate_selected_features)} features (Corr >= {quantile_value:.4f}).")
            
        except Exception as e:
            logger.error(f"Error during univariate filtering: {e}", exc_info=True)
            logger.warning("Skipping univariate filtering due to error.")
            X_train_univariate_filtered = X_train_clean # Use original cleaned data if filter fails
            
        if X_train_univariate_filtered.empty:
             logger.error("Feature set is empty after univariate filtering. Returning minimal whitelist.")
             return self.minimal_whitelist[:min_features]

        # --- 2. Logistic Regression (L1) Selection (Operates on univariately filtered data) --- # 
        logger.info(f"Performing L1 Logistic Regression (C={logreg_c}) on {X_train_univariate_filtered.shape[1]} features...")
        try:
            # Align target with the potentially reduced feature set again
            y_aligned_l1, X_aligned_l1 = y_dir_train.align(X_train_univariate_filtered, join='inner', axis=0)
            y_numeric_l1 = pd.to_numeric(y_aligned_l1, errors='coerce').fillna(0)

            if X_aligned_l1.empty:
                logger.error("Feature set empty after aligning for L1 step. Returning minimal whitelist.")
                return self.minimal_whitelist[:min_features]

            selector = SelectFromModel(
                LogisticRegression(
                    penalty='l1', 
                    C=logreg_c, 
                    solver='liblinear', # Good solver for L1
                    random_state=42,
                    max_iter=500 # Increased max_iter
                ), 
                threshold="mean" # Use mean threshold as suggested
            )
            selector.fit(X_aligned_l1, y_numeric_l1) # Fit on aligned data
            l1_selected_features = X_aligned_l1.columns[selector.get_support()].tolist()
            logger.info(f"L1 LogReg selected {len(l1_selected_features)} features (threshold='mean').")
            
            if not l1_selected_features:
                 logger.warning("L1 LogReg selected zero features after univariate filter. Returning minimal whitelist as fallback.")
                 return self.minimal_whitelist[:min_features] # Return min number from minimal list
                 
        except Exception as e:
             logger.error(f"Error during L1 Logistic Regression selection: {e}", exc_info=True)
             logger.warning("Falling back to minimal whitelist due to L1 selection error.")
             return self.minimal_whitelist[:min_features]

        # --- 3. VIF Calculation and Filtering (Operates on L1 selected features) --- #
        logger.info(f"Performing VIF filtering (threshold={vif_threshold}) on {len(l1_selected_features)} features...")
        features_to_check = l1_selected_features
        # Use the univariately filtered data to get the columns for VIF
        X_vif_input = X_train_univariate_filtered[features_to_check]

        final_whitelist = []
        while len(features_to_check) > min_features:
            X_subset = X_vif_input[features_to_check] # Subset the columns further in each iteration
            # Add constant for VIF calculation
            try:
                 # Align VIF input data again? Should be okay if features_to_check is correct.
                 X_subset_const = sm.add_constant(X_subset, has_constant='add')
                 vif = pd.Series(
                     [variance_inflation_factor(X_subset_const.values, i) 
                      for i in range(1, X_subset_const.shape[1])], # Skip constant (index 0)
                     index=X_subset.columns,
                     dtype=float # Ensure float type
                 )
                 
                 max_vif = vif.max()
                 if max_vif > vif_threshold:
                     feature_to_drop = vif.idxmax()
                     features_to_check.remove(feature_to_drop)
                     logger.debug(f"  Dropped '{feature_to_drop}' due to high VIF ({max_vif:.2f}) > {vif_threshold}. Remaining: {len(features_to_check)}")
                 else:
                     logger.debug(f"  Max VIF ({max_vif:.2f}) <= threshold ({vif_threshold}). Stopping VIF loop.")
                     break # All remaining features are below threshold
            except Exception as vif_e:
                 logger.error(f"Error calculating VIF for features {features_to_check}: {vif_e}", exc_info=True)
                 logger.warning("Aborting VIF filtering due to error.")
                 # Should we fallback? For now, just stop VIF filtering and use current set.
                 break
        
        final_whitelist = features_to_check

        # Ensure minimal whitelist features are included (if specified by config?)
        # Add back any minimal whitelist features that were dropped, if needed
        missing_minimal = set(self.minimal_whitelist) - set(final_whitelist)
        if missing_minimal:
            logger.info(f"Adding missing minimal whitelist features back: {list(missing_minimal)}")
            final_whitelist.extend(list(missing_minimal))
            # Ensure uniqueness after adding back
            final_whitelist = list(pd.Series(final_whitelist).unique())

        # Ensure minimum number of features
        if len(final_whitelist) < min_features:
             logger.warning(f"Final whitelist has {len(final_whitelist)} features, less than minimum required ({min_features}). Adding from minimal whitelist if possible.")
             needed = min_features - len(final_whitelist)
             additional_features = [f for f in self.minimal_whitelist if f not in final_whitelist][:needed]
             final_whitelist.extend(additional_features)
             logger.info(f"Added {len(additional_features)} features from minimal list. Final count: {len(final_whitelist)}")
             
        # --- Ensure bar_imputed is ALWAYS included --- #
        imputed_col_name = 'bar_imputed' # Assuming this is the column name
        if imputed_col_name not in final_whitelist:
            logger.warning(f"'{imputed_col_name}' was not selected by L1/VIF. Adding it back for sequence creation.")
            final_whitelist.append(imputed_col_name)
        # --- End Ensure --- #

        logger.info(f"Final whitelist contains {len(final_whitelist)} features after L1 + VIF selection.")
        logger.debug(f"Final Whitelist: {final_whitelist}")
        return final_whitelist

    def prune_features(self, df: pd.DataFrame, whitelist: list) -> pd.DataFrame:
        """
        Prunes the DataFrame to include only features in the whitelist.

        Args:
            df (pd.DataFrame): Input DataFrame.
            whitelist (list): List of feature names to keep.

        Returns:
            pd.DataFrame: Pruned DataFrame.
        """
        logger.info(f"Pruning features to whitelist ({len(whitelist)} features)...")
        missing_cols = [col for col in whitelist if col not in df.columns]
        if missing_cols:
            logger.warning(f"Whitelist features missing from DataFrame during pruning: {missing_cols}. These columns cannot be kept.")
            # Filter whitelist to only include existing columns
            whitelist = [col for col in whitelist if col in df.columns]
            
        if not whitelist: # Check if whitelist became empty
             logger.error("Whitelist is empty after checking for existing columns. Cannot prune.")
             return df # Return original df if whitelist is empty

        try:
            df_pruned = df[whitelist].copy()
            logger.info(f"Pruning complete. Shape after pruning: {df_pruned.shape}")
            return df_pruned
        except KeyError as e:
            logger.error(f"KeyError during pruning: {e}. Ensure whitelist columns exist in the DataFrame. Returning original DataFrame.")
            return df
        except Exception as e:
            logger.error(f"Unexpected error during pruning: {e}. Returning original DataFrame.")
            return df 

    def select_features_l1(self, X_train_scaled: pd.DataFrame, y_dir_train: pd.Series) -> List[str]:
        """Selects features using L1 regularization."""
        logger.info("Performing feature selection using L1 Logistic Regression (C=0.1)...")
        # Convert y_dir_train to int if it's not already (needed for LogReg)
        y_train_int = y_dir_train.astype(int)

        # Use configuration for C? For now, hardcoding C=0.1 as per log message
        # Added max_iter, verbose, and n_jobs=1 for debugging hangs
        C = 0.1
        logger.info(f"Performing feature selection using L1 Logistic Regression (C={C})...")
        # Ensure y_dir_train is 1D
        if y_dir_train.ndim > 1:
             y_dir_train = y_dir_train.squeeze()
             
        lr_l1 = LogisticRegression(
            C=C, 
            penalty='l1', 
            solver='liblinear', # Explicitly set solver
            random_state=42, 
            max_iter=1000, # Increase max_iter slightly just in case
            multi_class='auto' # Handle binary/multiclass automatically
        )

        try:
            logger.info(f"Input features shape for L1 selection: {X_train_scaled.shape}") # Log shape
            selector = SelectFromModel(lr_l1, threshold=-np.inf) # Keep all features with non-zero coef
            selector.fit(X_train_scaled, y_train_int)
            selected_mask = selector.get_support()
            selected_features = X_train_scaled.columns[selected_mask].tolist()
            
            # Ensure minimal whitelist features are included
            final_features = list(set(selected_features + self.minimal_whitelist))
            logger.info(f"L1 selected {len(selected_features)} features. After adding minimal whitelist, total: {len(final_features)}")
            return final_features
        except Exception as e:
            logger.error(f"Error during L1 feature selection: {e}", exc_info=True)
            # Fallback: return only the minimal whitelist
            logger.warning("Falling back to using only the minimal whitelist due to L1 error.")
            return self.minimal_whitelist 

    # <<< NEW METHOD: Price Range & Candle Features >>>
    def _add_candle_features(self, df: pd.DataFrame) -> pd.DataFrame:
        """Adds features based on candle shape and price range."""
        logger.info("Adding candle features...")
        required_cols = {'open', 'high', 'low', 'close'}
        if not required_cols.issubset(df.columns):
             logger.warning(f"Missing required columns for candle features ({required_cols - set(df.columns)}). Skipping.")
             return df

        df_shifted = df.shift(1)
        df_results = pd.DataFrame(index=df.index)

        try:
            # 1. Total Range
            df_results["total_range"] = df_shifted["high"] - df_shifted["low"]
            
            # 2. Body & Shadow Ratios
            body = df_shifted["close"] - df_shifted["open"]
            upper_shd = df_shifted["high"] - np.maximum(df_shifted["open"], df_shifted["close"])
            lower_shd = np.minimum(df_shifted["open"], df_shifted["close"]) - df_shifted["low"]
            
            range_with_eps = df_results["total_range"] + _EPS
            df_results["body_ratio"] = safe_divide(body, range_with_eps)
            df_results["upper_shd_ratio"] = safe_divide(upper_shd, range_with_eps)
            df_results["lower_shd_ratio"] = safe_divide(lower_shd, range_with_eps)
            
            # 3. Doji / Hammer flags
            df_results["is_doji"] = (np.abs(body) < 0.1 * df_results["total_range"]).astype(int)
            df_results["is_hammer"] = ((lower_shd > 2 * np.abs(body)) & (upper_shd < 0.1 * df_results["total_range"])).astype(int)

            # Fill NaNs
            candle_cols = ["total_range", "body_ratio", "upper_shd_ratio", "lower_shd_ratio", "is_doji", "is_hammer"]
            for col in candle_cols:
                 if col in df_results.columns:
                     # Fix inplace fillna
                     df_results[col] = df_results[col].ffill()
                     df_results[col] = df_results[col].bfill()
                     df_results[col] = df_results[col].fillna(0)

            # Add results to original DataFrame
            for col in candle_cols:
                 if col in df_results.columns:
                      df[col] = df_results[col]
                      
            logger.info(f"Successfully added candle features: {candle_cols}")

        except Exception as e:
             logger.error(f"Error adding candle features: {e}", exc_info=True)
             
        return df

    # <<< NEW METHOD: Volatility and Channel Features >>>
    def _add_volatility_channel_features(self, df: pd.DataFrame) -> pd.DataFrame:
        """Adds Bollinger Band Width and Keltner Channel Width."""
        logger.info("Adding volatility/channel features...")
        required_cols = {'high', 'low', 'close'} # Need these for base calcs
        if not required_cols.issubset(df.columns):
             logger.warning(f"Missing required columns for volatility/channel features. Skipping.")
             return df

        df_shifted = df.shift(1)
        df_results = pd.DataFrame(index=df.index)
        
        W = 20 # Common window for BBands/KC
        
        try:
            # Bollinger Bands Width (Normalized)
            try:
                bb_indicator = BollingerBands(close=df_shifted["close"], window=W, window_dev=2)
                bb_width = bb_indicator.bollinger_wband() # Width = (Upper - Lower)
                bb_center = bb_indicator.bollinger_mavg() # Center line (SMA)
                df_results["bb_width_norm"] = safe_divide(bb_width, bb_center, default=0.0)
            except Exception as bb_e:
                 logger.error(f"Error calculating Bollinger Band Width: {bb_e}")
                 df_results["bb_width_norm"] = 0.0
            
            # Keltner Channel Width (Normalized)
            try:
                # Ensure EMA_20 and ATR_14 are available (or calculate them)
                if f'EMA_{W}' not in df_shifted.columns:
                     ema_center = EMAIndicator(close=df_shifted["close"], window=W).ema_indicator()
                else:
                     ema_center = df_shifted[f'EMA_{W}']
                
                atr_window = 14 # Match ATR window used elsewhere? Or use W=20? Let's use 14 for now.
                if f'ATR_{atr_window}' not in df_shifted.columns:
                     atr_val = AverageTrueRange(high=df_shifted['high'], low=df_shifted['low'], close=df_shifted['close'], window=atr_window).average_true_range()
                else:
                     atr_val = df_shifted[f'ATR_{atr_window}']
                
                kc_dev_multiplier = 1.5 # Common multiplier
                kc_upper = ema_center + kc_dev_multiplier * atr_val
                kc_lower = ema_center - kc_dev_multiplier * atr_val
                kc_width = kc_upper - kc_lower
                df_results["kc_width_norm"] = safe_divide(kc_width, ema_center, default=0.0)
            except Exception as kc_e:
                 logger.error(f"Error calculating Keltner Channel Width: {kc_e}")
                 df_results["kc_width_norm"] = 0.0

            # Fill NaNs
            vol_cols = ["bb_width_norm", "kc_width_norm"]
            for col in vol_cols:
                 if col in df_results.columns:
                     # Fix inplace fillna
                     df_results[col] = df_results[col].ffill()
                     df_results[col] = df_results[col].bfill()
                     df_results[col] = df_results[col].fillna(0)

            # Add results to original DataFrame
            for col in vol_cols:
                 if col in df_results.columns:
                      df[col] = df_results[col]

            logger.info(f"Successfully added volatility/channel features: {vol_cols}")
            
        except ImportError:
             logger.error("'ta' library components missing. Cannot calculate BBands/KC features. Skipping.")
        except Exception as e:
             logger.error(f"Error calculating volatility/channel features: {e}", exc_info=True)
             
        return df

    # <<< NEW METHOD: Momentum and Trend Features >>>
    def _add_momentum_trend_features(self, df: pd.DataFrame) -> pd.DataFrame:
        """Adds Acceleration, Trend-Fit Residuals, Directional Change Count."""
        logger.info("Adding momentum/trend features...")
        # Requires return_1m, close
        if 'return_1m' not in df.columns or 'close' not in df.columns:
             logger.warning("Missing required 'return_1m' or 'close' columns. Skipping momentum/trend features.")
             return df
             
        df_shifted = df.shift(1) # Use shifted data for inputs when appropriate
        df_results = pd.DataFrame(index=df.index)

        try:
            # 1. Acceleration (using return_1m)
            df_results["accel_1m"] = df["return_1m"].diff() # Difference of the non-shifted return column

            # 2. Trend-Fit Residuals (3-bar rolling regression on shifted close price)
            df_results["trend_resid_3b"] = rolling_regression_residual(df_shifted['close'], window=3)

            # 3. Directional Change Count (consecutive up bars in last k=10 bars)
            k = 10
            up_seq = (df_shifted['close'] > df_shifted['close'].shift()).astype(int)
            # Calculate consecutive run length
            run_up = up_seq.groupby((up_seq != up_seq.shift()).cumsum()).cumcount() + 1
            # Zero out runs where the bar was not an up bar
            run_up_masked = run_up * up_seq
            df_results[f"dir_chg_cnt_{k}b"] = run_up_masked.rolling(window=k, min_periods=1).sum() # Sum of consecutive counts in window

            # Fill NaNs
            mom_cols = ["accel_1m", "trend_resid_3b", f"dir_chg_cnt_{k}b"]
            for col in mom_cols:
                 if col in df_results.columns:
                     # Fix inplace fillna
                     df_results[col] = df_results[col].ffill()
                     df_results[col] = df_results[col].bfill()
                     df_results[col] = df_results[col].fillna(0)

            # Add results to original DataFrame
            for col in mom_cols:
                 if col in df_results.columns:
                      df[col] = df_results[col]

            logger.info(f"Successfully added momentum/trend features: {mom_cols}")

        except Exception as e:
            logger.error(f"Error calculating momentum/trend features: {e}", exc_info=True)

        return df

    # <<< NEW METHOD: Additional Seasonality Features >>>
    def _add_extra_seasonality_features(self, df: pd.DataFrame) -> pd.DataFrame:
        """Adds Day-of-Week cycle and Time-to-Midnight features."""
        logger.info("Adding extra seasonality features...")
        if not isinstance(df.index, pd.DatetimeIndex):
            logger.warning("Index is not DatetimeIndex. Skipping extra seasonality features.")
            return df
            
        timestamp_source = df.index
        df_results = pd.DataFrame(index=df.index)

        try:
            # 1. Day-of-week cycle
            dow = timestamp_source.dayofweek # Monday=0, Sunday=6
            df_results["sin_dow"] = np.sin(2 * np.pi * dow / 7)
            df_results["cos_dow"] = np.cos(2 * np.pi * dow / 7)
            
            # 2. Time-to-midnight (UTC)
            minutes_in_day = 1440
            minutes_past_midnight = timestamp_source.hour * 60 + timestamp_source.minute
            minutes_to_midnight = minutes_in_day - minutes_past_midnight
            df_results["ttm_sin"] = np.sin(2 * np.pi * minutes_to_midnight / minutes_in_day)
            # df_results["ttm_cos"] = np.cos(2 * np.pi * minutes_to_midnight / minutes_in_day) # Optional Cosine component

            # Fill NaNs (shouldn't be any here, but for consistency)
            season_cols = ["sin_dow", "cos_dow", "ttm_sin"]
            for col in season_cols:
                 if col in df_results.columns:
                     # Fix inplace fillna (though likely unnecessary here)
                     df_results[col] = df_results[col].fillna(0)

            # Add results to original DataFrame
            for col in season_cols:
                 if col in df_results.columns:
                      df[col] = df_results[col]

            logger.info(f"Successfully added extra seasonality features: {season_cols}")
            
        except Exception as e:
            logger.error(f"Error calculating extra seasonality features: {e}", exc_info=True)

        return df 

    # <<< NEW METHOD: Interaction Features >>>
    def _add_interaction_features(self, df: pd.DataFrame) -> pd.DataFrame:
        """Adds interaction features based on existing base features."""
        logger.info("Adding interaction features...")
        df_results = pd.DataFrame(index=df.index)
        required_base_features = [
            # Group 1 deps
            "return_1m", "return_norm_1m", "volatility_14d",
            # Group 2 deps
            "svi_10", "chaikin_AD_10", "vol_norm_volume_spike",
            # Group 3 deps
            "return_60m",
            # Group 4 deps
            "hour_sin", "cos_dow",
            # Group 5 deps
            "EMA_10", "EMA_50", "body_ratio"
            # Group 6 deps are already listed
        ]
        missing_reqs = [f for f in required_base_features if f not in df.columns]
        if missing_reqs:
            logger.warning(f"Missing required base features for interactions: {missing_reqs}. Skipping interaction features.")
            return df

        try:
            # --- Group 1: Momentum x Volatility --- #
            # 1a: return_1m * rolling_std(return_1m, window=10)
            rolling_std_ret1m = df["return_1m"].rolling(window=10, min_periods=5).std()
            df_results["mom_vol_int_1m"] = df["return_1m"] * rolling_std_ret1m
            # 1b: return_norm_1m * volatility_14d
            df_results["norm_mom_vol_int_1m_14d"] = df["return_norm_1m"] * df["volatility_14d"]

            # --- Group 2: Price x Volume Imbalance --- #
            # 2a: SVI_10 * return_1m
            df_results["svi10_ret1m_int"] = df["svi_10"] * df["return_1m"]
            # 2b: Chaikin AD * Volume Spike (using 1440 window spike)
            df_results["ad10_volspike_int"] = df["chaikin_AD_10"] * df["vol_norm_volume_spike"]

            # --- Group 3: Cross-Horizon Momentum --- #
            # 3a: return_1m * return_60m
            df_results["ret1m_ret60m_int"] = df["return_1m"] * df["return_60m"]

            # --- Group 4: Seasonality x Edge --- #
            # 4a: hour_sin * return_1m
            df_results["hour_sin_ret1m_int"] = df["hour_sin"] * df["return_1m"]
            # 4b: cos_dow * return_60m
            df_results["dow_cos_ret60m_int"] = df["cos_dow"] * df["return_60m"]

            # --- Group 5: Trend x Candle Shape --- #
            # 5a: Trend Strength (EMA diff)
            df_results["ema_trend_strength"] = df["EMA_50"] - df["EMA_10"]
            # 5b: Body Ratio * Trend Strength
            df_results["body_ratio_trend_int"] = df["body_ratio"] * df_results["ema_trend_strength"]

            # --- Group 6: Auto-Polynomial Features --- #
            poly_features_in = ["return_1m", "volatility_14d", "svi_10"]
            missing_poly_in = [f for f in poly_features_in if f not in df.columns]
            if not missing_poly_in:
                logger.info(f"Generating polynomial features (degree=2, interaction_only) for: {poly_features_in}")
                poly = PolynomialFeatures(degree=2, interaction_only=True, include_bias=False)
                # Ensure input data is clean
                X_poly_in = df[poly_features_in].fillna(0) 
                X_poly_out = poly.fit_transform(X_poly_in)
                # Get feature names (will be like 'x0 x1', 'x0 x2', 'x1 x2')
                poly_feature_names_raw = poly.get_feature_names_out(poly_features_in)
                # Create more readable names
                poly_feature_names_out = [
                    f"{name.replace(' ', '_x_')}_poly" for name in poly_feature_names_raw
                ]
                # Add to results DataFrame
                df_poly = pd.DataFrame(X_poly_out, index=df.index, columns=poly_feature_names_out)
                # Add only the interaction terms (avoid duplicating originals)
                for col in df_poly.columns:
                    if col not in df_results.columns and col not in df.columns:
                        df_results[col] = df_poly[col]
                logger.info(f"Added polynomial features: {df_poly.columns.tolist()}")
            else:
                logger.warning(f"Skipping polynomial features: Missing input columns {missing_poly_in}")

            # --- Fill NaNs/Infs for all interaction features --- #
            interaction_cols = df_results.columns.tolist()
            logger.info(f"Generated interaction features: {interaction_cols}")
            for col in interaction_cols:
                 if col in df_results.columns:
                     # Fix inplace replace and fillna
                     df_results[col] = df_results[col].replace([np.inf, -np.inf], np.nan) # Use assignment
                     df_results[col] = df_results[col].ffill()
                     df_results[col] = df_results[col].bfill()
                     df_results[col] = df_results[col].fillna(0)
            
            # --- Add results to original DataFrame --- #
            for col in interaction_cols:
                 if col in df_results.columns:
                      df[col] = df_results[col]

        except Exception as e:
            logger.error(f"Error adding interaction features: {e}", exc_info=True)

        return df 