cj-dev-code/semantic_search

1from pathlib import Path2from langchain.text_splitter import RecursiveCharacterTextSplitter3 4def load_text(path):5    with open(path, "r", encoding="utf-8") as f:6        return f.read()7def save_chunks(chunks, out_path):8    with open(out_path, "w", encoding="utf-8") as f:9        for i, chunk in enumerate(chunks):10            f.write(f"--- Chunk {i} ---\n{chunk}\n\n")11 12 13def split_into_paragraph_chunks(text, chunk_size=1000, chunk_overlap=100):14    """15    Splits input text into paragraph-level chunks using RecursiveCharacterTextSplitter.16    17    - Prioritizes splitting on double newlines (paragraphs)18    - Falls back to single newline, then sentence, then word19    - Supports overlap to preserve context20 21    Args:22        text (str): The input text to split23        chunk_size (int): Maximum size of each chunk (in characters)24        chunk_overlap (int): Number of overlapping characters between chunks25 26    Returns:27        List[str]: List of text chunks28    """29    splitter = RecursiveCharacterTextSplitter(30        chunk_size=chunk_size,31        chunk_overlap=chunk_overlap,32        separators=["\n"]#, "\n", ".", " "]33    )34    return splitter.split_text(text)35 36if __name__ == "__main__":37    book_path = Path("data/fullbook.txt")38    out_path = Path("data/semantic_chunks.txt")39 40    text = load_text(book_path)41    chunks = split_into_paragraph_chunks(text)42    save_chunks(chunks, out_path)43