"""
Market data fetcher module for cryptocurrency market data from SQLite databases.

This module provides classes for fetching historical cryptocurrency market data
from SQLite databases downloaded from the crypto_md service. The raw data is always
stored in 1-minute intervals, and can be resampled to other intervals as needed.
"""

import os
import logging
import pandas as pd
import sqlite3
import glob
from datetime import datetime, timedelta
from typing import Dict, List, Tuple, Optional, Union
import re
import sys # Added

# V7: Update logger setup
# Configure logging with explicit console output
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
    handlers=[
        # logging.FileHandler("db_fetcher.log", mode='a'), # Optional file logging
        logging.StreamHandler(sys.stdout) 
    ]
)
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO) # Set level for this specific logger

class CryptoDBFetcher:
    """
    Fetches historical cryptocurrency market data from SQLite databases.
    
    The raw data in the SQLite databases is always stored in 1-minute intervals.
    This class can resample the data to other intervals (e.g., 5min, 1h, 1d) as needed.
    """
    
    # V7 Update: Adjusted defaults slightly, cache dir relative to project
    def __init__(self, db_dir: str = "downloaded_data", cache_dir: str = "data/cache", use_cache: bool = False):
        """
        Initialize the crypto database fetcher.
        
        Args:
            db_dir: Directory where SQLite database files are stored
            cache_dir: Directory to store cached data (relative to project root)
            use_cache: Whether to use cached data when available (default False for V7)
        """
        # V7 Update: Make db_dir potentially relative to workspace
        if not os.path.isabs(db_dir):
             # This assumes the script is run from the project root (e.g., v7/)
             # Adjust if running from elsewhere
             base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Go up two levels from src
             self.db_dir = os.path.join(base_path, db_dir)
        else:
             self.db_dir = db_dir
             
        self.cache_dir = cache_dir
        self.use_cache = use_cache
        
        if self.use_cache:
             os.makedirs(cache_dir, exist_ok=True)
        
        # Map of exchanges and pairs - discovered lazily now
        self._available_exchanges = None
        self._available_pairs = None
        self._db_files = None # Cache discovered DB files
        
        logger.info(f"Initialized CryptoDBFetcher with db_dir={self.db_dir}")

    @property
    def available_exchanges(self) -> List[str]:
        if self._available_exchanges is None:
            self._available_exchanges = self._discover_available_exchanges()
            logger.info(f"Discovered exchanges: {', '.join(self._available_exchanges)}")
        return self._available_exchanges

    @property
    def available_pairs(self) -> List[str]:
        if self._available_pairs is None:
            self._available_pairs = self._discover_available_pairs()
            logger.info(f"Discovered pairs: {', '.join(self._available_pairs)}")
        return self._available_pairs
    
    def _discover_available_exchanges(self) -> List[str]:
        """Discover available exchanges from database files."""
        exchanges = set()
        try:
            db_files = self._get_db_files()
            if not db_files: return []
            
            # Use the first DB file to discover exchanges
            with sqlite3.connect(db_files[0]) as conn:
                tables = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE '%_ohlcv_%'").fetchall()
                for table in tables:
                    table_name = table[0]
                    try:
                        # Attempt 1: Extract from table name (e.g., coinbase_ohlcv_1min)
                        match = re.match(r'([a-z0-9]+)_ohlcv_', table_name, re.IGNORECASE)
                        if match:
                            exchanges.add(match.group(1).upper())
                        # Attempt 2: Query distinct exchange_id if column exists
                        elif 'exchange_id' in [c[1].lower() for c in conn.execute(f"PRAGMA table_info({table_name})").fetchall()]:
                            query = f"SELECT DISTINCT exchange_id FROM {table_name}"
                            for row in conn.execute(query).fetchall(): exchanges.add(row[0].upper())
                    except sqlite3.Error as e: logger.debug(f"Could not query table {table_name}: {e}")
            
            return sorted(list(exchanges))
        except Exception as e: logger.error(f"Error discovering exchanges: {e}"); return []
    
    def _discover_available_pairs(self) -> List[str]:
        """Discover available trading pairs from database files."""
        pairs = set()
        try:
            db_files = self._get_db_files()
            if not db_files: return []
            
            with sqlite3.connect(db_files[0]) as conn:
                tables = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE '%_ohlcv_%'").fetchall()
                for table in tables:
                    table_name = table[0]
                    try:
                         # Check if instrument_id column exists
                         if 'instrument_id' in [c[1].lower() for c in conn.execute(f"PRAGMA table_info({table_name})").fetchall()]:
                             query = f"SELECT DISTINCT instrument_id FROM {table_name}"
                             for row in conn.execute(query).fetchall():
                                 instr_id = row[0]
                                 if instr_id.startswith("PAIR-"): instr_id = instr_id[5:]
                                 pairs.add(instr_id)
                         else:
                              # Attempt to parse from table name if instrument_id col missing
                              # e.g., coinbase_ohlcv_1min_btc_usdt
                              match = re.search(r'_([A-Z0-9]+_[A-Z0-9]+)$|_([A-Z0-9]+-[A-Z0-9]+)$', table_name, re.IGNORECASE)
                              if match:
                                   pair = match.group(1) or match.group(2)
                                   if pair: pairs.add(pair.replace('_','-').upper())
                    except sqlite3.Error as e: logger.debug(f"Could not query table {table_name}: {e}")
            return sorted(list(pairs))
        except Exception as e: logger.error(f"Error discovering pairs: {e}"); return []
    
    def _get_db_files(self) -> List[str]:
        """Get available database files, sorted by date desc (cached)."""
        if self._db_files is not None:
             return self._db_files
             
        logger.info(f"Scanning for DB files in: {self.db_dir}")
        if not os.path.exists(self.db_dir):
            logger.error(f"Database directory {self.db_dir} does not exist")
            self._db_files = []
            return []
        
        patterns = ["*.mktdata.ohlcv.db", "*.db", "*.sqlite", "*.sqlite3"]
        db_files = []
        for pattern in patterns:
            files = glob.glob(os.path.join(self.db_dir, pattern))
            if files: logger.debug(f"Found {len(files)} files with pattern {pattern}")
            db_files.extend(files)
        
        if not db_files:
            logger.warning(f"No database files found in {self.db_dir} matching patterns: {patterns}")
            self._db_files = []
            return []
        
        db_files = list(set(db_files)) # Remove duplicates
        
        # Sort by date (newest first) if possible
        date_pattern = re.compile(r'(\d{8})')
        file_dates = []
        for file in db_files:
            basename = os.path.basename(file)
            match = date_pattern.search(basename)
            date_obj = None
            if match:
                try: date_obj = pd.to_datetime(match.group(1), format='%Y%m%d')
                except ValueError: pass
            # Fallback: try modification time
            if date_obj is None:
                 try: date_obj = pd.to_datetime(os.path.getmtime(file), unit='s')
                 except Exception: date_obj = pd.Timestamp.min # Default to oldest if error
            file_dates.append((date_obj, file))
            
        file_dates.sort(key=lambda x: x[0], reverse=True) # Sort by date object, newest first
        self._db_files = [file for _, file in file_dates]
        
        logger.info(f"Found {len(self._db_files)} DB files. Using newest: {os.path.basename(self._db_files[0]) if self._db_files else 'None'}")
        return self._db_files
    
    # V7 Update: Simplified date finding - use files covering the range
    def _get_relevant_db_files(self, start_dt: pd.Timestamp, end_dt: pd.Timestamp) -> List[str]:
        """Find DB files potentially containing data for the date range."""
        all_files = self._get_db_files()
        relevant_files = set()
        date_pattern = re.compile(r'(\d{8})')
        
        for file in all_files:
            basename = os.path.basename(file)
            match = date_pattern.search(basename)
            if match:
                try:
                    file_date = pd.to_datetime(match.group(1), format='%Y%m%d')
                    # Check if the file's date is within or overlaps the target range 
                    # (Assume file contains data for that single day)
                    if start_dt.date() <= file_date.date() <= end_dt.date():
                         relevant_files.add(file)
                except ValueError:
                    pass # Ignore files with unparseable dates
            else:
                 # If no date in filename, conservatively include recent files 
                 # based on modification time (might be less accurate)
                 try:
                      mod_time = pd.to_datetime(os.path.getmtime(file), unit='s')
                      # Include if modified within or shortly after the requested range
                      if start_dt <= mod_time <= (end_dt + timedelta(days=1)):
                           relevant_files.add(file)
                 except Exception:
                      pass # Ignore files with mod time errors
        
        # If no files found based on date, return the most recent one as a fallback
        if not relevant_files and all_files:
            logger.warning(f"No DB files found matching date range {start_dt.date()} - {end_dt.date()}. Using most recent file.")
            return [all_files[0]]
        elif not relevant_files:
             logger.error("No relevant DB files found and no fallback files available.")
             return []
             
        # Sort the relevant files chronologically (oldest first for processing)
        return sorted(list(relevant_files), key=lambda f: os.path.basename(f)) 

    def _convert_ticker_to_instrument_id(self, ticker: str) -> str:
        if not ticker.startswith("PAIR-"): return f"PAIR-{ticker}"
        return ticker
    
    def _convert_interval(self, interval: str) -> Optional[str]:
        interval = interval.lower()
        interval_map = {"1m": "1min", "5m": "5min", "15m": "15min", "30m": "30min",
                        "1h": "1hour", "4h": "4hour", "1d": "1day", "1D": "1day"}
        if interval in interval_map: return interval_map[interval]
        if interval.endswith(('min', 'hour', 'day')): return interval
        logger.warning(f"Unsupported interval format: {interval}")
        return None # Return None for unsupported intervals
    
    def _get_table_name(self, conn: sqlite3.Connection, exchange: str, interval: str = "1min") -> Optional[str]:
        """Find the correct table name, trying variations."""
        base_table = f"{exchange.lower()}_ohlcv_{interval}"
        cursor = conn.cursor()
        cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
        tables = [row[0] for row in cursor.fetchall()]
        
        if base_table in tables: return base_table
        
        # Try variations (case, interval format)
        variations = [
            f"{exchange.upper()}_ohlcv_{interval}",
            f"{exchange.lower()}_ohlcv_1m", # Always check 1min as source
            f"{exchange.upper()}_ohlcv_1m",
        ]
        for var in variations:
            if var in tables: 
                logger.debug(f"Found table using variation: {var}")
                return var
        
        # Check if any OHLCV table exists for the exchange
        for t in tables:
             if t.lower().startswith(f"{exchange.lower()}_ohlcv_"):
                  logger.warning(f"Using first available OHLCV table for exchange: {t}")
                  return t
                  
        logger.warning(f"Table for {exchange} interval {interval} not found.")
        return None
        
    def _query_data_from_db(self, db_file: str, ticker: str, start_timestamp: int, end_timestamp: int, 
                            interval: str = "1min", exchange: str = "COINBASE") -> pd.DataFrame:
        """
        Query market data from a database file.
        """
        instrument_id = self._convert_ticker_to_instrument_id(ticker)
        # Always query 1min interval from DB as it's the source resolution
        query_interval = "1min"
        
        try:
            # logger.debug(f"Querying DB {db_file} for {instrument_id}...")
            with sqlite3.connect(db_file) as conn:
                table_name = self._get_table_name(conn, exchange, query_interval)
                if not table_name:
                    return pd.DataFrame()
                
                cursor = conn.cursor()
                cursor.execute(f"PRAGMA table_info({table_name})")
                columns_info = cursor.fetchall()
                column_names = [col[1].lower() for col in columns_info]
                # logger.debug(f"Columns in {table_name}: {column_names}")
                
                # Build query dynamically
                select_cols = ["tstamp", "open", "high", "low", "close", "volume"]
                select_str = ", ".join([c for c in select_cols if c in column_names])
                if not all(c in column_names for c in ["tstamp", "open", "high", "low", "close", "volume"]):
                     logger.warning(f"Table {table_name} in {db_file} missing standard OHLCV columns.")
                     # Attempt to map common variations if needed - skipped for simplicity
                     # For now, return empty if standard columns are missing
                     return pd.DataFrame()
                
                where_clauses = ["tstamp >= ?", "tstamp <= ?"]
                params = [start_timestamp, end_timestamp]
                
                if 'instrument_id' in column_names:
                    where_clauses.append("instrument_id = ?")
                    params.append(instrument_id)
                if 'exchange_id' in column_names:
                     # Normalize exchange name from DB if needed
                     cursor.execute(f"SELECT DISTINCT exchange_id FROM {table_name} WHERE exchange_id LIKE ? LIMIT 1", (f'%{exchange}%',))
                     db_exchange = cursor.fetchone()
                     if db_exchange:
                          params.append(db_exchange[0])
                          where_clauses.append("exchange_id = ?")
                     else: # If exchange not found, query might return empty
                          params.append(exchange)
                          where_clauses.append("exchange_id = ?") 
                          
                query = f"SELECT {select_str} FROM {table_name} WHERE {' AND '.join(where_clauses)} ORDER BY tstamp"
                # logger.debug(f"Executing query: {query} with params {params[:2]}...{params[-1]}")
                
                df = pd.read_sql_query(query, conn, params=params)
                if df.empty: return pd.DataFrame()

                # Convert timestamp and set index
                df['date'] = pd.to_datetime(df['tstamp'], unit='ns', utc=True)
                df = df.set_index('date').drop(columns=['tstamp']) # Drop original tstamp
                # logger.debug(f"Query returned {len(df)} rows.")
                return df
                
        except Exception as e:
            logger.error(f"Error querying {db_file} table {table_name if 'table_name' in locals() else 'N/A'}: {e}", exc_info=False)
            return pd.DataFrame()
    
    def _resample_data(self, df: pd.DataFrame, interval: str) -> pd.DataFrame:
        """Resample 1-minute data to a different interval."""
        if df.empty or not isinstance(df.index, pd.DatetimeIndex): return df
        
        # V7 Update: More robust interval conversion
        try: 
            freq = pd.tseries.frequencies.to_offset(interval)
            if freq is None: raise ValueError("Invalid frequency")
        except ValueError:
             # Try manual mapping for common cases
             interval_map = {"1min": "1min", "5min": "5min", "15min": "15min", "30min": "30min",
                             "1hour": "1h", "4hour": "4h", "1day": "1d"}
             if interval in interval_map: freq = interval_map[interval]
             else: logger.error(f"Unsupported interval for resampling: {interval}"); return df
        
        logger.info(f"Resampling data to {freq}...")
        try:
            agg_dict = {'open': 'first', 'high': 'max', 'low': 'min', 'close': 'last'}
            # Only include volume if present
            if 'volume' in df.columns: agg_dict['volume'] = 'sum'
            
            # Check for required columns
            missing_cols = [c for c in ['open','high','low','close'] if c not in df.columns]
            if missing_cols:
                 logger.error(f"Cannot resample, missing required columns: {missing_cols}")
                 return pd.DataFrame() # Return empty if essential cols missing
                 
            resampled = df.resample(freq).agg(agg_dict)
            resampled = resampled.dropna(subset=['open', 'high', 'low', 'close']) # Drop rows where OHLC couldn't be computed
            logger.info(f"Resampling complete. New shape: {resampled.shape}")
            
        except Exception as e:
            logger.error(f"Error during resampling to {freq}: {e}", exc_info=True)
            return pd.DataFrame() # Return empty on error
        
        return resampled
    
    def fetch_data(self, ticker: str, start_date: str = None, end_date: str = None, interval: str = "1min",
                  exchange: str = "COINBASE") -> pd.DataFrame:
        """
        Fetch cryptocurrency market data for a given ticker and date range.
        Always sources 1-minute data from DB and resamples if needed.
        """
        logger.info(f"Fetching {ticker} data from {start_date} to {end_date} at {interval} (Exchange: {exchange})...")
        
        # V7 Update: Stricter date handling
        try:
            start_dt = pd.to_datetime(start_date, utc=True) if start_date else pd.Timestamp.now(tz='utc') - timedelta(days=30)
            end_dt = pd.to_datetime(end_date, utc=True) if end_date else pd.Timestamp.now(tz='utc')
            if start_dt >= end_dt: raise ValueError("Start date must be before end date")
        except Exception as e: logger.error(f"Invalid date format/range: {e}"); return pd.DataFrame()
        logger.info(f"Querying date range: {start_dt.date()} to {end_dt.date()}")
        
        # V7 Update: Check supported interval format
        target_interval = interval # Store requested interval
        resample_freq_pd = None
        try: resample_freq_pd = pd.tseries.frequencies.to_offset(target_interval)
        except ValueError: 
             interval_map = {"1min": "1T", "5min": "5T", "15min": "15T", "30min": "30T",
                             "1hour": "1H", "4hour": "4H", "1day": "1D"}
             if target_interval in interval_map: resample_freq_pd = interval_map[target_interval]
             else: logger.error(f"Unsupported interval: {target_interval}"); return pd.DataFrame()
        
        # V7 Update: Cache key includes exchange
        cache_key = f"{exchange}_{ticker}_{start_dt.strftime('%Y%m%d')}_{end_dt.strftime('%Y%m%d')}_{target_interval}".replace("-","_")
        cache_path = os.path.join(self.cache_dir, f"{cache_key}.parquet") # Use parquet for better type handling
        
        if self.use_cache and os.path.exists(cache_path):
            logger.info(f"Loading data from cache: {cache_path}")
            try: 
                data = pd.read_parquet(cache_path)
                # Ensure index is datetime and UTC (Parquet often preserves this)
                if not isinstance(data.index, pd.DatetimeIndex): data.index = pd.to_datetime(data.index)
                if data.index.tz is None: data.index = data.index.tz_localize('utc')
                elif data.index.tz != 'UTC': data.index = data.index.tz_convert('utc')
                logger.info(f"Loaded {len(data)} rows from cache.")
                return data
            except Exception as e: logger.warning(f"Error loading cache: {e}. Fetching fresh data.")
        
        # Convert timestamps for DB query
        start_timestamp_ns = int(start_dt.timestamp() * 1e9)
        end_timestamp_ns = int(end_dt.timestamp() * 1e9)
        
        # V7 Update: Use _get_relevant_db_files
        db_files_to_query = self._get_relevant_db_files(start_dt, end_dt)
        if not db_files_to_query:
             logger.error("No relevant database files found for the specified date range.")
             return pd.DataFrame()
        logger.info(f"Querying {len(db_files_to_query)} DB files: {[os.path.basename(f) for f in db_files_to_query]}")
        
        all_data = []
        for db_file in db_files_to_query:
            # Always query 1min data
            df = self._query_data_from_db(db_file, ticker, start_timestamp_ns, end_timestamp_ns, "1min", exchange)
            if not df.empty: all_data.append(df)
        
        if not all_data: logger.warning(f"No data found in DBs for {ticker}..."); return pd.DataFrame()
        
        combined_df = pd.concat(all_data)
        combined_df = combined_df[~combined_df.index.duplicated(keep='first')].sort_index()
        # Filter exact date range AFTER combining, before resampling
        combined_df = combined_df[(combined_df.index >= start_dt) & (combined_df.index <= end_dt)]
        logger.info(f"Combined data shape before resampling: {combined_df.shape}")
        
        # Resample if target interval is not 1 minute
        final_df = combined_df
        if target_interval != "1min":
            final_df = self._resample_data(combined_df, target_interval)
            if final_df.empty:
                 logger.error("Resampling resulted in empty DataFrame.")
                 return pd.DataFrame()
        
        # Cache the result
        if self.use_cache:
            try:
                 os.makedirs(os.path.dirname(cache_path), exist_ok=True)
                 final_df.to_parquet(cache_path)
                 logger.info(f"Saved {len(final_df)} rows to cache: {cache_path}")
            except Exception as e: logger.warning(f"Failed to save cache: {e}")
        
        logger.info(f"Fetch data complete. Returning {len(final_df)} rows.")
        return final_df
    
    # ... (keep _extract_date_from_filename) ...
    def _extract_date_from_filename(self, filename):
        if not filename: return None
        base_name = os.path.basename(filename)
        match = re.match(r"(\d{8})\.mktdata", base_name)
        if match: 
            try: return datetime.strptime(match.group(1), "%Y%m%d").date()
            except ValueError: return None
        return None

# --- Removed fetch_intraday_data, fetch_daily_data, batch_fetch_data, DataFetcherFactory --- 
# These methods are not directly used by the V7 workflow as defined so far.
# They can be added back if needed. 