codekingpro/portable-devtools

1from typing import List2 3from langchain_core.documents import Document4 5from langchain_community.document_loaders.web_base import WebBaseLoader6 7 8class AZLyricsLoader(WebBaseLoader):9    """Load `AZLyrics` webpages."""10 11    def load(self) -> List[Document]:12        """Load webpages into Documents."""13        soup = self.scrape()14        title = soup.title.text15        lyrics = soup.find_all("div", {"class": ""})[2].text16        text = title + lyrics17        metadata = {"source": self.web_path}18        return [Document(page_content=text, metadata=metadata)]19