codekingpro/portable-devtools

1"""JavaScript framework text splitter."""2 3import re4from typing import Any5 6from langchain_text_splitters import RecursiveCharacterTextSplitter7 8 9class JSFrameworkTextSplitter(RecursiveCharacterTextSplitter):10    """Text splitter that handles React (JSX), Vue, and Svelte code.11 12    This splitter extends `RecursiveCharacterTextSplitter` to handle React (JSX), Vue,13    and Svelte code by:14 15    1. Detecting and extracting custom component tags from the text16    2. Using those tags as additional separators along with standard JS syntax17 18    The splitter combines:19 20    * Custom component tags as separators (e.g. `<Component`, `<div`)21    * JavaScript syntax elements (function, const, if, etc)22    * Standard text splitting on newlines23 24    This allows chunks to break at natural boundaries in React, Vue, and Svelte25    component code.26    """27 28    def __init__(29        self,30        separators: list[str] | None = None,31        chunk_size: int = 2000,32        chunk_overlap: int = 0,33        **kwargs: Any,34    ) -> None:35        """Initialize the JS Framework text splitter.36 37        Args:38            separators: Optional list of custom separator strings to use39            chunk_size: Maximum size of chunks to return40            chunk_overlap: Overlap in characters between chunks41            **kwargs: Additional arguments to pass to parent class42        """43        super().__init__(chunk_size=chunk_size, chunk_overlap=chunk_overlap, **kwargs)44        self._separators = separators or []45 46    def split_text(self, text: str) -> list[str]:47        """Split text into chunks.48 49        This method splits the text into chunks by:50 51        * Extracting unique opening component tags using regex52        * Creating separators list with extracted tags and JS separators53        * Splitting the text using the separators by calling the parent class method54 55        Args:56            text: String containing code to split57 58        Returns:59            List of text chunks split on component and JS boundaries60        """61        # Extract unique opening component tags using regex62        # Regex to match opening tags, excluding self-closing tags63        opening_tags = re.findall(r"<\s*([a-zA-Z0-9]+)[^>]*>", text)64 65        component_tags = []66        for tag in opening_tags:67            if tag not in component_tags:68                component_tags.append(tag)69        component_separators = [f"<{tag}" for tag in component_tags]70 71        js_separators = [72            "\nexport ",73            " export ",74            "\nfunction ",75            "\nasync function ",76            " async function ",77            "\nconst ",78            "\nlet ",79            "\nvar ",80            "\nclass ",81            " class ",82            "\nif ",83            " if ",84            "\nfor ",85            " for ",86            "\nwhile ",87            " while ",88            "\nswitch ",89            " switch ",90            "\ncase ",91            " case ",92            "\ndefault ",93            " default ",94        ]95        # Build the effective separator list for this call only.96        # Do NOT assign back to self._separators: doing so would permanently97        # append js_separators + component_separators on every invocation,98        # causing the list to grow unboundedly when split_text() is called99        # multiple times on the same instance.100        separators = (101            self._separators102            + js_separators103            + component_separators104            + ["<>", "\n\n", "&&\n", "||\n"]105        )106        return self._split_text(text, separators)107