codekingpro/portable-devtools

1import logging2from typing import Callable, List, Optional3 4from langchain_core._api.deprecation import deprecated5from langchain_core.documents import Document6 7from langchain_community.document_loaders.base import BaseLoader8from langchain_community.document_loaders.gcs_file import GCSFileLoader9from langchain_community.utilities.vertexai import get_client_info10 11logger = logging.getLogger(__name__)12 13 14@deprecated(15    since="0.0.32",16    removal="1.0",17    alternative_import="langchain_google_community.GCSDirectoryLoader",18)19class GCSDirectoryLoader(BaseLoader):20    """Load from GCS directory."""21 22    def __init__(23        self,24        project_name: str,25        bucket: str,26        prefix: str = "",27        loader_func: Optional[Callable[[str], BaseLoader]] = None,28        continue_on_failure: bool = False,29    ):30        """Initialize with bucket and key name.31 32        Args:33            project_name: The ID of the project for the GCS bucket.34            bucket: The name of the GCS bucket.35            prefix: The prefix of the GCS bucket.36            loader_func: A loader function that instantiates a loader based on a37                file_path argument. If nothing is provided, the  GCSFileLoader38                would use its default loader.39            continue_on_failure: To use try-except block for each file within the GCS40                directory. If set to `True`, then failure to process a file will not41                cause an error.42        """43        self.project_name = project_name44        self.bucket = bucket45        self.prefix = prefix46        self._loader_func = loader_func47        self.continue_on_failure = continue_on_failure48 49    def load(self) -> List[Document]:50        """Load documents."""51        try:52            from google.cloud import storage53        except ImportError:54            raise ImportError(55                "Could not import google-cloud-storage python package. "56                "Please install it with `pip install google-cloud-storage`."57            )58        client = storage.Client(59            project=self.project_name,60            client_info=get_client_info(module="google-cloud-storage"),61        )62        docs = []63        for blob in client.list_blobs(self.bucket, prefix=self.prefix):64            # we shall just skip directories since GCSFileLoader creates65            # intermediate directories on the fly66            if blob.name.endswith("/"):67                continue68            # Use the try-except block here69            try:70                loader = GCSFileLoader(71                    self.project_name,72                    self.bucket,73                    blob.name,74                    loader_func=self._loader_func,75                )76                docs.extend(loader.load())77            except Exception as e:78                if self.continue_on_failure:79                    logger.warning(f"Problem processing blob {blob.name}, message: {e}")80                    continue81                else:82                    raise e83        return docs84