Processors API

class DataProcessor:
    """Main processor class that handles processing of all data types."""

    _type_processors: Dict[DataType, Type[BaseTypeProcessor]] = {}

    # Define defaults for insulin classification centrally
    DEFAULT_INSULIN_BOLUS_LIMIT: float = 8.0
    DEFAULT_INSULIN_MAX_DOSE: float = 15.0

    @staticmethod
    def create_table_configs(
        detected_format: DeviceFormat,
    ) -> Dict[str, TableStructure]:
        """
        Creates a table configuration dictionary from detected format.

        Args:
            detected_format: Format object containing files and their table configurations

        Returns:
            Dict[str, TableStructure]: Dictionary mapping table names to their structures
        """
        try:
            return {
                table.name: table
                for file_config in detected_format.files
                for table in file_config.tables
            }
        except Exception as e:
            logger.error("Failed to create table configurations: %s", str(e))
            raise ProcessingError("Failed to create table configurations") from e

    def process_tables(
        self,
        table_data: Dict[str, TableData],
        detected_format: DeviceFormat,
        interpolation_limit: Optional[
            Any
        ] = None,  # Optional parameter for CGM processor
        bolus_limit: Optional[
            float
        ] = None,  # Optional parameters for insulin processor
        max_dose: Optional[float] = None,
    ) -> Dict[DataType, ProcessedTypeData]:
        """
        Process all tables according to their configuration.

        Args:
            table_data: Dictionary mapping table names to their data
            detected_format: Format object containing table configurations
            interpolation_limit: Max length of gaps to interpolate
            bolus_limit: Maximum insulin dose to be classified as bolus(default = 8)
            max_dose: Maximum insulin dose - all over will be discarded

        Returns:
            Dict[DataType, ProcessedTypeData]: Processed data organized by type
        """
        table_configs = self.create_table_configs(detected_format)

        # Rest of your existing process_tables implementation
        type_data: Dict[DataType, List[ColumnData]] = {}

        for table_name, data in table_data.items():
            config = table_configs[table_name]

            # Group columns by data type
            for column in config.columns:
                if column.data_type:
                    # Include insulin meta data with insulin data
                    target_type = (
                        DataType.INSULIN
                        if column.data_type == DataType.INSULIN_META
                        else column.data_type
                    )

                    df_subset = data.dataframe[[column.source_name]].copy()
                    df_subset.columns = ["value"]

                    column_data = ColumnData(
                        dataframe=df_subset,
                        unit=column.unit,
                        config=column,
                        is_primary=column.is_primary,
                    )

                    if target_type not in type_data:
                        type_data[target_type] = []

                    type_data[target_type].append(column_data)

        # Process each data type
        results = {}
        for data_type, columns in type_data.items():
            try:
                processor = self.get_processor_for_type(data_type)

                # Inject optional parameters based on processor type
                if data_type == DataType.CGM and interpolation_limit is not None:
                    result = processor.process_type(columns, interpolation_limit)
                elif data_type == DataType.INSULIN:
                    # Use provided values or defaults, but never None
                    final_bolus_limit = (
                        bolus_limit
                        if bolus_limit is not None
                        else self.DEFAULT_INSULIN_BOLUS_LIMIT
                    )
                    final_max_dose = (
                        max_dose
                        if max_dose is not None
                        else self.DEFAULT_INSULIN_MAX_DOSE
                    )
                    result = processor.process_type(
                        columns, final_bolus_limit, final_max_dose
                    )
                else:
                    result = processor.process_type(columns)

                if not result.dataframe.empty:
                    results[data_type] = result

                    col_count = len(columns)
                    primary_count = sum(1 for c in columns if c.is_primary)
                    logger.info(
                        "    \u2713 Processed %s: %d primary and %d secondary columns",
                        data_type.name,
                        primary_count,
                        col_count - primary_count,
                    )

            except ProcessingError as e:
                logger.error("Error processing %s: %s", data_type, str(e))
                continue

        return results

    @classmethod
    def register_processor(cls, data_type: DataType):
        """Register a processor class for a specific data type."""

        def wrapper(processor_cls: Type[BaseTypeProcessor]):
            cls._type_processors[data_type] = processor_cls
            return processor_cls

        return wrapper

    def get_processor_for_type(self, data_type: DataType) -> BaseTypeProcessor:
        """Get appropriate processor instance for the data type."""
        processor_cls = self._type_processors.get(data_type)
        if processor_cls is None:
            raise ProcessingError(
                f"No processor registered for data type: {data_type.value}"
            )
        return processor_cls()

@DataProcessor.register_processor(DataType.BGM)
class BGMProcessor(BaseTypeProcessor):
    """Processes BGM data with validation and cleaning."""

    def process_type(
        self,
        columns: List[ColumnData],
    ) -> ProcessedTypeData:
        """Process all BGM data from various sources.

        Args:
            columns: List of ColumnData containing all BGM data columns

        Returns:
            ProcessedTypeData containing combined and cleaned BGM data
        """
        processing_notes = []

        try:
            # Validate we have at least one primary column
            if not any(col.is_primary for col in columns):
                raise ProcessingError("No primary BGM column found")

            # Create initial dataframe
            combined_df = pd.DataFrame(index=pd.DatetimeIndex([]))
            column_units = {}

            # First pass - combine all data and convert units
            for idx, col_data in enumerate(
                sorted(columns, key=lambda x: (not x.is_primary))
            ):
                # Convert to mg/dL if needed
                df = col_data.dataframe.copy()

                # Filter out invalid/zero values before unit conversion
                valid_mask = df["value"] > 0.0
                invalid_count = (~valid_mask).sum()
                if invalid_count > 0:
                    processing_notes.append(
                        f"Removed {invalid_count} invalid/zero values from column {idx + 1}"
                    )
                df = df[valid_mask]

                if col_data.unit == Unit.MMOL:
                    df["value"] = df["value"] * 18.0182
                    processing_notes.append(
                        f"Converted BGM column {idx + 1} from {Unit.MMOL.value} to {Unit.MGDL.value}"
                    )

                # Generate column name
                new_name = self._generate_column_name(
                    DataType.BGM, col_data.is_primary, idx
                )
                df.columns = [new_name]

                # Add clipped flag column before any clipping occurs
                clipped_name = f"{new_name}_clipped"
                df[clipped_name] = False

                # Identify values outside valid range
                below_range = df[new_name] < 39.64
                above_range = df[new_name] > 360.36

                # Update clipped flags
                df.loc[below_range | above_range, clipped_name] = True

                # Clip values to valid range
                df[new_name] = df[new_name].clip(lower=39.64, upper=360.36)

                # Log clipping statistics
                clipped_count = (below_range | above_range).sum()
                if clipped_count > 0:
                    processing_notes.append(
                        f"Clipped {clipped_count} values in column {new_name} "
                        f"({below_range.sum()} below range, {above_range.sum()} above range)"
                    )

                # Merge with existing data
                if combined_df.empty:
                    combined_df = df
                else:
                    combined_df = combined_df.join(df, how="outer")

                column_units[new_name] = Unit.MGDL

            # Handle duplicate timestamps by keeping the last value
            duplicates = combined_df.index.duplicated(keep="last")
            if duplicates.any():
                dup_count = duplicates.sum()
                processing_notes.append(
                    f"Found {dup_count} duplicate timestamps - keeping last values"
                )
                combined_df = combined_df[~duplicates]

            # Create mmol/L columns for each mg/dL column (excluding clipped flag columns)
            value_columns = [
                col for col in combined_df.columns if not col.endswith("_clipped")
            ]
            for col in value_columns:
                mmol_col = f"{col}_mmol"
                combined_df[mmol_col] = combined_df[col] * 0.0555
                column_units[mmol_col] = Unit.MMOL

            # Track stats about the processed data
            total_readings = len(combined_df)
            readings_per_day = (
                total_readings
                / (
                    (combined_df.index.max() - combined_df.index.min()).total_seconds()
                    / 86400
                )
                if total_readings > 0
                else 0
            )

            processing_notes.extend(
                [
                    f"Processed {total_readings} total BGM readings",
                    f"Average of {readings_per_day:.1f} readings per day",
                    f"Date range: {combined_df.index.min()} to {combined_df.index.max()}",
                ]
            )

            return ProcessedTypeData(
                dataframe=combined_df,
                source_units=column_units,
                processing_notes=processing_notes,
            )

        except Exception as e:
            raise ProcessingError(f"Error processing BGM data: {str(e)}") from e

@DataProcessor.register_processor(DataType.CARBS)
class CarbsProcessor(BaseTypeProcessor):
    """Processes carbohydrate intake data with validation and cleaning."""

    def process_type(
        self,
        columns: List[ColumnData],
    ) -> ProcessedTypeData:
        """Process all carbohydrate data from various sources.

        Args:
            columns: List of ColumnData containing all carb data columns

        Returns:
            ProcessedTypeData containing combined and cleaned carb data
        """
        processing_notes = []

        try:
            # Validate we have at least one primary column
            if not any(col.is_primary for col in columns):
                raise ProcessingError("No primary carbohydrate column found")

            # Sort columns to ensure primary is first
            sorted_columns = sorted(columns, key=lambda x: (not x.is_primary))

            # Combine all columns with standardized names
            combined_df, column_units = self._combine_and_rename_columns(
                sorted_columns, DataType.CARBS
            )

            if combined_df.empty:
                raise ProcessingError("No carbohydrate data to process")

            # Log what we're processing
            processing_notes.append(
                f"Processing {len(combined_df.columns)} carb columns: {', '.join(combined_df.columns)}"
            )

            # Track original row count
            original_count = len(combined_df)

            # Process each carb column
            for col in combined_df.columns:
                # Keep only rows where carbs is >= 1.0 grams
                mask = combined_df[col] >= 1.0
                combined_df.loc[~mask, col] = None

                filtered_count = mask.sum()
                processing_notes.append(
                    f"Column {col}: Kept {filtered_count} entries ≥1g "
                    f"({filtered_count / original_count * 100:.1f}%)"
                )

            # Drop rows where all values are null (no significant carbs in any column)
            combined_df = combined_df.dropna(how="all")

            # Handle duplicate timestamps by keeping the first occurrence
            duplicates = combined_df.index.duplicated()
            if duplicates.any():
                dup_count = duplicates.sum()
                processing_notes.append(f"Removed {dup_count} duplicate timestamps")
                combined_df = combined_df[~duplicates]

            # Final stats
            processing_notes.append(
                f"Final dataset contains {len(combined_df)} entries "
                f"from {original_count} original records"
            )

            return ProcessedTypeData(
                dataframe=combined_df,
                source_units=column_units,
                processing_notes=processing_notes,
            )

        except Exception as e:
            raise ProcessingError(
                f"Error processing carbohydrate data: {str(e)}"
            ) from e

@DataProcessor.register_processor(DataType.CGM)
class CGMProcessor(BaseTypeProcessor):
    """Processes CGM data with validation and cleaning."""

    def process_type(
        self,
        columns: List[ColumnData],
        interpolation_limit: int = 4,
    ) -> ProcessedTypeData:
        """Process all CGM data from various sources.

        Args:
            columns: List of ColumnData containing all CGM data columns
            interpolation_limit: Maximum number of consecutive missing values to interpolate
                               (default: 4, equivalent to 20 minutes at 5-min intervals)

        Returns:
            ProcessedTypeData containing combined and cleaned CGM data
        """
        processing_notes = []

        try:
            # Validate we have at least one primary column
            if not any(col.is_primary for col in columns):
                raise ProcessingError("No primary CGM column found")

            # Sort columns to ensure primary is first
            sorted_columns = sorted(columns, key=lambda x: (not x.is_primary))

            # Create initial dataframe
            combined_df = pd.DataFrame(index=pd.DatetimeIndex([]))
            column_units = {}

            # First pass - combine all data and convert units
            for idx, col_data in enumerate(sorted_columns):
                # Convert to mg/dL if needed
                df = col_data.dataframe.copy()
                if col_data.unit == Unit.MMOL:
                    df["value"] = df["value"] * 18.0182
                    processing_notes.append(
                        f"Converted CGM column {idx + 1} from {Unit.MMOL.value} to {Unit.MGDL.value}"
                    )

                # Generate column name
                new_name = self._generate_column_name(
                    DataType.CGM, col_data.is_primary, idx
                )
                df.columns = [new_name]

                # Merge with existing data
                if combined_df.empty:
                    combined_df = df
                else:
                    combined_df = combined_df.join(df, how="outer")

                column_units[new_name] = col_data.unit

            # Round all timestamps to nearest 5 minute interval
            combined_df.index = combined_df.index.round("5min")

            # Handle duplicate times by taking mean
            combined_df = combined_df.groupby(level=0).mean()

            # Create complete 5-minute interval index
            full_index = pd.date_range(
                start=combined_df.index.min(), end=combined_df.index.max(), freq="5min"
            )

            # Reindex to include all intervals
            combined_df = combined_df.reindex(full_index)

            # Get primary column name
            primary_col = next(col for col in combined_df.columns if "primary" in col)

            # Create missing flags for each column
            missing_flags = pd.DataFrame(index=combined_df.index)
            missing_flags["missing"] = combined_df[primary_col].isna()

            # Handle interpolation for each column
            for col in combined_df.columns:
                # Create groups of consecutive missing values
                gap_groups = (~combined_df[col].isna()).cumsum()

                # Within each False group (where missing=True), count the group size
                gap_size = (
                    combined_df[combined_df[col].isna()].groupby(gap_groups).size()
                )

                # Identify gap groups that are larger than interpolation_limit
                large_gaps = gap_size[gap_size > interpolation_limit].index

                # Interpolate all gaps initially
                combined_df[col] = combined_df[col].interpolate(
                    method="linear",
                    limit=interpolation_limit,
                    limit_direction="forward",
                )

                # Reset interpolated values back to NaN for large gaps
                for gap_group in large_gaps:
                    mask = (gap_groups == gap_group) & combined_df[col].isna()
                    combined_df.loc[mask, col] = np.nan

                # Clip values to valid range
                combined_df[col] = combined_df[col].clip(lower=39.64, upper=360.36)

            # Create mmol/L columns for each mg/dL column
            for col in combined_df.columns.copy():
                mmol_col = f"{col}_mmol"
                combined_df[mmol_col] = combined_df[col] * 0.0555
                column_units[mmol_col] = Unit.MMOL
                column_units[col] = Unit.MGDL

            # Add the missing flags
            combined_df["missing"] = missing_flags["missing"]

            # Track stats about the processed data
            total_readings = len(combined_df)
            missing_primary = combined_df["missing"].sum()
            total_na = combined_df["cgm_primary"].isna().sum()
            initial_completeness_percent = (
                (total_readings - missing_primary) / total_readings
            ) * 100
            remaining_completeness_percent = (
                (total_readings - total_na) / total_readings
            ) * 100
            processing_notes.extend(
                [
                    f"Processed {total_readings} total CGM readings",
                    f"Found {missing_primary} missing or interpolated values in primary data",
                    f"Found {total_na} missing values after interpolation",
                    f"Initial CGM dataset completeness: {initial_completeness_percent:.2f}%",
                    f"CGM completeness after interpolation: {remaining_completeness_percent:.2f}%",
                ]
            )

            return ProcessedTypeData(
                dataframe=combined_df,
                source_units=column_units,
                processing_notes=processing_notes,
            )

        except Exception as e:
            raise ProcessingError(f"Error processing CGM data: {str(e)}") from e