codekingpro/portable-devtools

1from typing import Iterator, Optional2 3from langchain_community.docstore.document import Document4from langchain_community.document_loaders.base import BaseLoader5from langchain_community.document_loaders.unstructured import UnstructuredFileIOLoader6 7 8class AzureAIDataLoader(BaseLoader):9    """Load from Azure AI Data."""10 11    def __init__(self, url: str, glob: Optional[str] = None):12        """Initialize with URL to a data asset or storage location13        ."""14        self.url = url15        """URL to the data asset or storage location."""16        self.glob_pattern = glob17        """Optional glob pattern to select files. Defaults to None."""18 19    def lazy_load(self) -> Iterator[Document]:20        """A lazy loader for Documents."""21        try:22            from azureml.fsspec import AzureMachineLearningFileSystem23        except ImportError as exc:24            raise ImportError(25                "Could not import azureml-fspec package."26                "Please install it with `pip install azureml-fsspec`."27            ) from exc28 29        fs = AzureMachineLearningFileSystem(self.url)30 31        if self.glob_pattern:32            remote_paths_list = fs.glob(self.glob_pattern)33        else:34            remote_paths_list = fs.ls()35 36        for remote_path in remote_paths_list:37            with fs.open(remote_path) as f:38                loader = UnstructuredFileIOLoader(file=f)39                yield from loader.load()40