codekingpro/portable-devtools

1from typing import Any, Iterator, Literal2 3from langchain_core.documents import Document4 5from langchain_community.document_loaders.base import BaseLoader6 7 8class BaseDataFrameLoader(BaseLoader):9    def __init__(self, data_frame: Any, *, page_content_column: str = "text"):10        """Initialize with dataframe object.11 12        Args:13            data_frame: DataFrame object.14            page_content_column: Name of the column containing the page content.15              Defaults to "text".16        """17        self.data_frame = data_frame18        self.page_content_column = page_content_column19 20    def lazy_load(self) -> Iterator[Document]:21        """Lazy load records from dataframe."""22 23        for _, row in self.data_frame.iterrows():24            metadata = row.to_dict()25            text = metadata.pop(self.page_content_column)26            yield Document(page_content=text, metadata=metadata)27 28 29class DataFrameLoader(BaseDataFrameLoader):30    """Load `Pandas` DataFrame."""31 32    def __init__(33        self,34        data_frame: Any,35        page_content_column: str = "text",36        engine: Literal["pandas", "modin"] = "pandas",37    ):38        """Initialize with dataframe object.39 40        Args:41            data_frame: Pandas DataFrame object.42            page_content_column: Name of the column containing the page content.43              Defaults to "text".44        """45        try:46            if engine == "pandas":47                import pandas as pd48            elif engine == "modin":49                import modin.pandas as pd50            else:51                raise ValueError(52                    f"Unsupported engine {engine}. Must be one of 'pandas', or 'modin'."53                )54        except ImportError as e:55            raise ImportError(56                "Unable to import pandas, please install with `pip install pandas`."57            ) from e58 59        if not isinstance(data_frame, pd.DataFrame):60            raise ValueError(61                f"Expected data_frame to be a pd.DataFrame, got {type(data_frame)}"62            )63        super().__init__(data_frame, page_content_column=page_content_column)64