Underground-Digital/Workflow-Engine

1from core.rag.extractor.extractor_base import BaseExtractor2from core.rag.models.document import Document3from services.website_service import WebsiteService4 5 6class JinaReaderWebExtractor(BaseExtractor):7    """8    Crawl and scrape websites and return content in clean llm-ready markdown.9    """10 11    def __init__(self, url: str, job_id: str, tenant_id: str, mode: str = "crawl", only_main_content: bool = False):12        """Initialize with url, api_key, base_url and mode."""13        self._url = url14        self.job_id = job_id15        self.tenant_id = tenant_id16        self.mode = mode17        self.only_main_content = only_main_content18 19    def extract(self) -> list[Document]:20        """Extract content from the URL."""21        documents = []22        if self.mode == "crawl":23            crawl_data = WebsiteService.get_crawl_url_data(self.job_id, "jinareader", self._url, self.tenant_id)24            if crawl_data is None:25                return []26            document = Document(27                page_content=crawl_data.get("content", ""),28                metadata={29                    "source_url": crawl_data.get("url"),30                    "description": crawl_data.get("description"),31                    "title": crawl_data.get("title"),32                },33            )34            documents.append(document)35        return documents36