codekingpro/portable-devtools

1from typing import List2 3from langchain_core.documents import Document4 5from langchain_community.document_loaders.base import BaseLoader6 7 8class GutenbergLoader(BaseLoader):9    """Load from `Gutenberg.org`."""10 11    def __init__(self, file_path: str):12        """Initialize with a file path."""13        if not file_path.startswith("https://www.gutenberg.org"):14            raise ValueError("file path must start with 'https://www.gutenberg.org'")15 16        if not file_path.endswith(".txt"):17            raise ValueError("file path must end with '.txt'")18 19        self.file_path = file_path20 21    def load(self) -> List[Document]:22        """Load file."""23        from urllib.request import urlopen24 25        elements = urlopen(self.file_path)26        text = "\n\n".join([str(el.decode("utf-8-sig")) for el in elements])27        metadata = {"source": self.file_path}28        return [Document(page_content=text, metadata=metadata)]29