Team Ai
Apppublic

CintraAI/code-chunker

sourceHugging Facemitupdated 1y agoView on Hugging Face
5likes
test_code_chunker.cpython-312.pyc67 linesDownload Raw Back to __pycache__
1�

2Ǹ�f�3��x�ddlZddlmZddlmZmZddlZddlmZddede	fd�Z3Gd�d	ej�ZGd4�dej�Z
Gd�d
ej�ZGd�dej�ZGd�dej�ZGd�dej�Zedk(rej&�yy)�N)�patch)�Chunker�CodeChunker)�	load_json�string�returnc�d�tj|�}t|j|��}|S)z.Returns the number of tokens in a text string.)�tiktoken�encoding_for_model�len�encode)r�
encoding_name�encoding�5num_tokenss    �8c:\Users\joesh\.vscode\code-chunker\test_code_chunker.py�mock_count_tokensr6s+���*�*�=�9�H��X�_�_�V�,�-�J���c�B�eZdZd�Zd�Zd�Zd�Zd�Zd�Zd�Z	d�Z7d	�Zy8)�TestCodeChunkerPythonc��tdt��|_|jj�|_t	d��|_t
d�|_y)Nzutils.count_tokens��side_effect�py��file_extension�mock_codefiles.json�rr�patcher�startr�code_chunkerr�
mock_codebase��selfs r�setUpzTestCodeChunkerPython.setUpsB���1�?P�Q���!%���!3�!3�!5���'�t�<���&�'<�=��rc�8�|jj�y�N�r�stopr"s r�tearDownzTestCodeChunkerPython.tearDown��������rc��|jd}td�}td|���|jj	|d��}|j|�}td|���td|���tj|�tj|�}td|���tj|�}|j|t|jd	���|j||�|jt|�d9�|jd|d�|jd|d10�y)
Nz	simple.pyz11import syszfirst_chunk_token_limit = ���token_limitztoken_count = zoriginal code:12 zcode after consolidation:13 �14��zprint('Hello, world!'))
r!r�printr �chunkr�print_chunks�consolidate_chunks_into_file�count_lines�assertEqualr�split�assertIn)r#�py_code�first_chunk_token_limit�chunks�token_count�	full_code�	num_liness       r�test_chunk_simple_codez,TestCodeChunkerPython.test_chunk_simple_codes���$�$�[�1��"3�L�"A��
�*�+B�*C�D�E��"�"�(�(��b�(�A���,�,�W�5��
��{�m�,�-�
�!�'��+�,����V�$��8�8��@�	�
�,�Y�K�8�9��'�'�	�2�	�����C��
�
�d�(;�$<�=��
�
�i��)�����V��a�(��
�
�l�F�1�I�.��
�
�.��q�	�:rc���|jd}|jj|d��}tj|�tj15|�}tjtj16|��}|j|t|jd���|j||�|jt|�d�|jd|d�y)Nztext_only.py�r-r/r1zNThis file is empty and should test the chunker's ability to handle empty files�r!r r3rr4r5r6r7rr8r9�r#r:r<�17final_coder?s     r�test_chunk_code_text_onlyz/TestCodeChunkerPython.test_chunk_code_text_only-s����$�$�^�4���"�"�(�(��b�(�A�����V�$��9�9�&�A�18��'�'��(L�(L�V�(T�U�	�����C��
�
�d�(;�$<�=��
�
�g�z�*�����V��a�(��
�
�f�hn�op�hq�rrc�x�|jd}|jj|d��}tj|�tj19|�}tjtj20|��}|j|t|jd���|j||�y)Nz	routes.pyrBr-r/rCrDs     r�test_chunk_code_with_routesz1TestCodeChunkerPython.test_chunk_code_with_routes9����$�$�[�1���"�"�(�(��b�(�A�����V�$��9�9�&�A�21��'�'��(L�(L�V�(T�U�	�����C��
�
�d�(;�$<�=��
�
�g�z�*rc�x�|jd}|jj|d��}tj|�tj22|�}tjtj23|��}|j|t|jd���|j||�y)Nz	models.pyrBr-r/rCrDs     r�test_chunk_code_with_modelsz1TestCodeChunkerPython.test_chunk_code_with_modelsCrIrc�x�|jd}|jj|d��}tj|�tj24|�}tjtj25|��}|j|t|jd���|j||�y)Nzmain.pyrBr-r/rCrDs     r�test_chunk_code_with_mainz/TestCodeChunkerPython.test_chunk_code_with_mainL����$�$�Y�/���"�"�(�(��b�(�A�����V�$��9�9�&�A�26��'�'��(L�(L�V�(T�U�	�����C��
�
�d�(;�$<�=��
�
�g�z�*rc�x�|jd}|jj|d��}tj|�tj27|�}tjtj28|��}|j|t|jd���|j||�y)Nzutilities.pyrBr-r/rCrDs     r�test_chunk_code_with_utilitiesz4TestCodeChunkerPython.test_chunk_code_with_utilitiesU����$�$�^�4���"�"�(�(��b�(�A�����V�$��9�9�&�A�29��'�'��(L�(L�V�(T�U�	�����C��
�
�d�(;�$<�=��
�
�g�z�*rc�x�|jd}|jj|d��}tj|�tj30|�}tjtj31|��}|j|t|jd���|j||�y)Nzbig_class.pyrBr-r/rCrDs     r�test_chunk_code_with_big_classz4TestCodeChunkerPython.test_chunk_code_with_big_class^rQrN)�__name__�32__module__�__qualname__r$r)r@rFrHrKrMrPrS�rrrrs0��>��;�$	s�+�+�+�+�+rrc�B�eZdZd�Zd�Zd�Zd�Zd�Zd�Zd�Z	d�Z33d	�Zy34)�TestCodeChunkerJavaScriptc��tdt��|_|jj�|_t	d��|_t
d�|_y)N�)app.util.TextChunker.Chunker.count_tokensr�jsrrrr"s rr$zTestCodeChunkerJavaScript.setUpj�B���H�Vg�h���!%���!3�!3�!5���'�t�<���&�'<�=��rc�8�|jj�yr&r'r"s rr)z"TestCodeChunkerJavaScript.tearDownpr*rc�x�|jd}|jj|d��}tj|�tj35|�}tjtj36|��}|j|t|jd���|j||�y)Nz	simple.jsrBr-r/rC�r#�js_coder<rEr?s     r�!test_chunk_javascript_simple_codez;TestCodeChunkerJavaScript.test_chunk_javascript_simple_codesrIrc�x�|jd}|jj|d��}tj|�tj37|�}tjtj38|��}|j|t|jd���|j||�y)Nz	routes.jsrBr-r/rCr`s     r�!test_chunk_javascript_with_routesz;TestCodeChunkerJavaScript.test_chunk_javascript_with_routes}rIrc�x�|jd}|jj|d��}tj|�tj39|�}tjtj40|��}|j|t|jd���|j||�y)Nz	models.jsrBr-r/rCr`s     r�!test_chunk_javascript_with_modelsz;TestCodeChunkerJavaScript.test_chunk_javascript_with_models�rIrc�x�|jd}|jj|d��}tj|�tj41|�}tjtj42|��}|j|t|jd���|j||�y)Nzmain.jsrBr-r/rCr`s     r�test_chunk_javascript_with_mainz9TestCodeChunkerJavaScript.test_chunk_javascript_with_main�rNrc�x�|jd}|jj|d��}tj|�tj43|�}tjtj44|��}|j|t|jd���|j||�y)Nzutilities.jsrBr-r/rCr`s     r�$test_chunk_javascript_with_utilitiesz>TestCodeChunkerJavaScript.test_chunk_javascript_with_utilities�rQrc�x�|jd}|jj|d��}tj|�tj45|�}tjtj46|��}|j|t|jd���|j||�y)Nzbig_class.jsrBr-r/rCr`s     r�$test_chunk_javascript_with_big_classz>TestCodeChunkerJavaScript.test_chunk_javascript_with_big_class�rQrc�x�|jd}|jj|d��}tj|�tj47|�}tjtj48|��}|j|t|jd���|j||�y)Nzreact_component.jsrBr-r/rCr`s     r�*test_chunk_javascript_with_react_componentzDTestCodeChunkerJavaScript.test_chunk_javascript_with_react_component�s����$�$�%9�:���"�"�(�(��b�(�A�����V�$��9�9�&�A�49��'�'��(L�(L�V�(T�U�	�����C��
�
�d�(;�$<�=��
�
�g�z�*rN)rTrUrVr$r)rbrdrfrhrjrlrnrWrrrYrYhs/��>��+�+�+�+�+�+�+rrYc�$�eZdZd�Zd�Zd�Zd�Zy)�TestCodeChunkerCSSc��tdt��|_|jj�|_t	d��|_t
d�|_y)Nr[r�cssrrrr"s rr$zTestCodeChunkerCSS.setUp�sB���H�Vg�h���!%���!3�!3�!5���'�u�=���&�'<�=��rc�8�|jj�yr&r'r"s rr)zTestCodeChunkerCSS.tearDown�r*rc�x�|jd}|jj|d��}tj|�tj50|�}tjtj51|��}|j|t|jd���|j||�y)Nzmedia_queries.cssrBr-r/rC�r#�css_coder<rEr?s     r�test_chunk_css_with_media_queryz2TestCodeChunkerCSS.test_chunk_css_with_media_query�����%�%�&9�:���"�"�(�(��r�(�B�����V�$��9�9�&�A�52��'�'��(L�(L�V�(T�U�	�����C����t�(<�$=�>��
�
�h�53�+rc�x�|jd}|jj|d��}tj|�tj54|�}tjtj55|��}|j|t|jd���|j||�y)Nzsimple_styles.cssrBr-r/rCrus     r�test_chunk_css_with_simple_cssz1TestCodeChunkerCSS.test_chunk_css_with_simple_css�rxrN)rTrUrVr$r)rwrzrWrrrprp�s��>��,�,rrpc��eZdZd�Zd�Zd�Zy)�TestCodeChunkerTypeScriptc��tdt��|_|jj�|_t	d��|_t
d�|_y)Nr[r�tsrrrr"s rr$zTestCodeChunkerTypeScript.setUp�r]rc�8�|jj�yr&r'r"s rr)z"TestCodeChunkerTypeScript.tearDown�r*rc��|jd}|jj|d��}tj|�tj56|�}tj|�}|j|t|jd���|j||�|jt|�d�y)Nz57example.tsrBr-r/r1�r!r r3rr4r5r6r7rr8r9�
assertGreater)r#�ts_coder<rEr?s     r�test_chunk_typescript_codez4TestCodeChunkerTypeScript.test_chunk_typescript_code�����$�$�\�2���"�"�(�(��b�(�A�����V�$��9�9�&�A�58��'�'�59�3�	�����C��
�
�d�(;�$<�=��
�
�g�z�*����3�v�;��*rN)rTrUrVr$r)r�rWrrr|r|����>��+rr|c��eZdZd�Zd�Zd�Zy)�TestCodeChunkerRubyc��tdt��|_|jj�|_t	d��|_t
d�|_y)Nr[r�rbrrrr"s rr$zTestCodeChunkerRuby.setUp�r]rc�8�|jj�yr&r'r"s rr)zTestCodeChunkerRuby.tearDown�r*rc��|jd}|jj|d��}tj|�tj60|�}tj|�}|j|t|jd���|j||�|jt|�d�y)Nz61example.rbrBr-r/r1r�)r#�rb_coder<rEr?s     r�test_chunk_ruby_codez(TestCodeChunkerRuby.test_chunk_ruby_code�r�rN)rTrUrVr$r)r�rWrrr�r��r�rr�c��eZdZd�Zd�Zd�Zy)�TestCodeChunkerPHPc��tdt��|_|jj�|_t	d��|_t
d�|_y)Nr[r�phprrrr"s rr$zTestCodeChunkerPHP.setUpsB���H�Vg�h���!%���!3�!3�!5���'�u�=���&�'<�=��rc�8�|jj�yr&r'r"s rr)zTestCodeChunkerPHP.tearDownr*rc��|jd}|jj|d��}tj|�tj62|�}tj|�}|j|t|jd���|j||�|jt|�d�y)Nzexample.phprBr-r/r1r�)r#�php_coder<rEr?s     r�test_chunk_php_codez&TestCodeChunkerPHP.test_chunk_php_codes����%�%�m�4���"�"�(�(��r�(�B�����V�$��9�9�&�A�63��'�'�64�3�	�����C����t�(<�$=�>��
�
�h�65�+����3�v�;��*rN)rTrUrVr$r)r�rWrrr�r�s��>��+rr��__main__)zgpt-4)�unittest�
unittest.mockrrrr66�utilsr�str�intr�TestCaserrYrpr|r�r�rT�mainrWrr�<module>r�s�����(���67�c��S��T+�H�-�-�T+�nJ+�� 1� 1�J+�Z,��*�*�,�B+�� 1� 1�+�.+�(�+�+�+�.+��*�*�+�*�z���H�M�M�O�r