Underground-Digital/Workflow-Engine

1import logging2 3from core.rag.extractor.extractor_base import BaseExtractor4from core.rag.models.document import Document5 6logger = logging.getLogger(__name__)7 8 9class UnstructuredTextExtractor(BaseExtractor):10    """Load msg files.11 12 13    Args:14        file_path: Path to the file to load.15    """16 17    def __init__(self, file_path: str, api_url: str):18        """Initialize with file path."""19        self._file_path = file_path20        self._api_url = api_url21 22    def extract(self) -> list[Document]:23        from unstructured.partition.text import partition_text24 25        elements = partition_text(filename=self._file_path)26        from unstructured.chunking.title import chunk_by_title27 28        chunks = chunk_by_title(elements, max_characters=2000, combine_text_under_n_chars=2000)29        documents = []30        for chunk in chunks:31            text = chunk.text.strip()32            documents.append(Document(page_content=text))33 34        return documents35