AlainDeLong/End-To-End-Machine-Learning-Project

1import os2import sys3from dataclasses import dataclass4 5import numpy as np6import pandas as pd7from sklearn.pipeline import Pipeline8from sklearn.compose import ColumnTransformer9from sklearn.impute import SimpleImputer10from sklearn.preprocessing import OneHotEncoder, StandardScaler11 12from src.logger import logging13from src.exception import CustomException14from src.utils import save_object15 16 17@dataclass18class DataTransformationConfig:19    preprocessor_obj_file_path = os.path.join("artifacts", "preprocessor.pkl")20 21 22class DataTransformation:23    def __init__(self) -> None:24        self.data_transformation_config = DataTransformationConfig()25 26    def get_data_transformer_object(self):27        """28        This function is responsible for data transformation29        """30        try:31            numerical_columns = ["reading_score", "writing_score"]32            catogrical_columns = [33                "gender",34                "race_ethnicity",35                "parental_level_of_education",36                "lunch",37                "test_preparation_course",38            ]39 40            num_pipeline = Pipeline(41                steps=[42                    ("imputer", SimpleImputer(strategy="median")),43                    ("scaler", StandardScaler()),44                ]45            )46            logging.info("Numerical columns standard scaling completed")47 48            cat_pipeline = Pipeline(49                steps=[50                    ("imputer", SimpleImputer(strategy="most_frequent")),51                    ("one_hot_encoder", OneHotEncoder()),52                    # ("scaler", StandardScaler()),53                ]54            )55            logging.info("Categorical columns encoding completed")56 57            logging.info(f"Numerical columns: {numerical_columns}")58            logging.info(f"Categorical columns: {catogrical_columns}")59 60            preprocessor = ColumnTransformer(61                transformers=[62                    ("num_pipeline", num_pipeline, numerical_columns),63                    ("cat_pipeline", cat_pipeline, catogrical_columns),64                ]65            )66 67            return preprocessor68 69        except Exception as e:70            raise CustomException(e, sys)71 72    def initiate_data_transformation(self, train_path, test_path):73        try:74            train_df = pd.read_csv(train_path)75            test_df = pd.read_csv(test_path)76            logging.info("Read train and test data completed")77            logging.info("Obtaining preprocessing object")78 79            preprocessing_obj = self.get_data_transformer_object()80 81            target_column_name = "math_score"82            # numerical_columns = (["reading_score", "writing_score"],)83 84            input_feature_train_df = train_df.drop(columns=[target_column_name], axis=1)85            target_feature_train_df = train_df[target_column_name]86 87            input_feature_test_df = test_df.drop(columns=[target_column_name], axis=1)88            target_feature_test_df = test_df[target_column_name]89 90            logging.info(91                f"Applying preprocessing object on training and testing dataframe"92            )93 94            input_feature_train_arr = preprocessing_obj.fit_transform(95                input_feature_train_df96            )97            input_feature_test_arr = preprocessing_obj.transform(input_feature_test_df)98 99            train_arr = np.c_[100                input_feature_train_arr, np.array(target_feature_train_df)101            ]102            test_arr = np.c_[input_feature_test_arr, np.array(target_feature_test_df)]103 104            save_object(105                file_path=self.data_transformation_config.preprocessor_obj_file_path,106                obj=preprocessing_obj,107            )108            logging.info(f"Saved preprocessing object")109 110            return (111                train_arr,112                test_arr,113                self.data_transformation_config.preprocessor_obj_file_path,114            )115        except Exception as e:116            raise CustomException(e, sys)117