codekingpro/portable-devtools

1"""Konlpy text splitter."""2 3from __future__ import annotations4 5from typing import Any6 7from typing_extensions import override8 9from langchain_text_splitters.base import TextSplitter10 11try:12    import konlpy13 14    _HAS_KONLPY = True15except ImportError:16    _HAS_KONLPY = False17 18 19class KonlpyTextSplitter(TextSplitter):20    """Splitting text using Konlpy package.21 22    It is good for splitting Korean text.23    """24 25    def __init__(26        self,27        separator: str = "\n\n",28        **kwargs: Any,29    ) -> None:30        """Initialize the Konlpy text splitter.31 32        Args:33            separator: The separator to use when combining splits.34 35        Raises:36            ImportError: If Konlpy is not installed.37        """38        super().__init__(**kwargs)39        self._separator = separator40        if not _HAS_KONLPY:41            msg = """42                Konlpy is not installed, please install it with43                `pip install konlpy`44                """45            raise ImportError(msg)46        self.kkma = konlpy.tag.Kkma()47 48    @override49    def split_text(self, text: str) -> list[str]:50        splits = self.kkma.sentences(text)51        return self._merge_splits(splits, self._separator)52