codekingpro/portable-devtools

1import logging2from pathlib import Path3from typing import Iterator, Optional, Union4 5from langchain_core.documents import Document6 7from langchain_community.document_loaders.base import BaseLoader8from langchain_community.document_loaders.helpers import detect_file_encodings9 10logger = logging.getLogger(__name__)11 12 13class TextLoader(BaseLoader):14    """Load text file.15 16 17    Args:18        file_path: Path to the file to load.19 20        encoding: File encoding to use. If `None`, the file will be loaded21        with the default system encoding.22 23        autodetect_encoding: Whether to try to autodetect the file encoding24            if the specified encoding fails.25    """26 27    def __init__(28        self,29        file_path: Union[str, Path],30        encoding: Optional[str] = None,31        autodetect_encoding: bool = False,32    ):33        """Initialize with file path."""34        self.file_path = file_path35        self.encoding = encoding36        self.autodetect_encoding = autodetect_encoding37 38    def lazy_load(self) -> Iterator[Document]:39        """Load from file path."""40        text = ""41        try:42            with open(self.file_path, encoding=self.encoding) as f:43                text = f.read()44        except UnicodeDecodeError as e:45            if self.autodetect_encoding:46                detected_encodings = detect_file_encodings(self.file_path)47                for encoding in detected_encodings:48                    logger.debug(f"Trying encoding: {encoding.encoding}")49                    try:50                        with open(self.file_path, encoding=encoding.encoding) as f:51                            text = f.read()52                        break53                    except UnicodeDecodeError:54                        continue55            else:56                raise RuntimeError(f"Error loading {self.file_path}") from e57        except Exception as e:58            raise RuntimeError(f"Error loading {self.file_path}") from e59 60        metadata = {"source": str(self.file_path)}61        yield Document(page_content=text, metadata=metadata)62