Team Ai
Apppublic

CintraAI/code-chunker

sourceHugging Facemitupdated 1y agoView on Hugging Face
5likes
CodeParser.cpython-312.pyc71 linesDownload Raw Back to __pycache__
1�

2+��f"6��|�ddlZddlZddlmZmZmZmZddlmZm	Z	m3Z4ddlmZmZddlZdedefd�Z
Gd�d	�Zy)5�N)�List�Dict�Union�Tuple)�Language�Parser�Node)rr�code�returnc��|jd�}t|�D��cgc]\}}d|dz�d|����}}}dj|�}|Scc}}w)N�6zLine ��: )�split�	enumerate�join)r7�8code_lines�i�line�code_with_line_numbers�joined_liness      �1c:\Users\joesh\.vscode\code-chunker\CodeParser.py�$return_simple_line_numbers_with_codersY�����D�!�J�EN�z�EZ�[�'�!�T��a�!�e�W�B�t�f�5�[��[��9�9�3�4�L����\s�Ac	�b�eZdZejjd�Zddedee	e	ffd�Z9d�Zde	de	defd	�Z
d10e	de	dedeffd�Zd
ede	deeee	ffd�Zde	dee	e	ffd�Zde	dee	e	ffd�Zd
ede	deeee	ffd�Zd11e	de	deefd�Zd12e	de	deefd�Zd13e	de	fd�Zdd�Zy)�14CodeParserz~/.code_parser_cacheN�file_extensionsc	��t|t�r|g}ddddddddd�|_|�g|_n<|D�cgc]+}||jvr|jj	|���-c}|_i|_|j
�ycc}w)N�python�15javascript�css�16typescript�php�ruby)�py�js�jsxr �ts�tsxr"�rb)�17isinstance�str�language_extension_map�language_names�get�	languages�_install_parsers)�selfr�exts   r�__init__zCodeParser.__init__s����o�s�+�.�/�O���������	'18��#��"�"$�D��Sb�#G�C�#&�$�*E�*E�#E�$(�#>�#>�#B�#B�3�#G�#G�D����������#Gs�0Bc19���tjtjd��	tjj|j�stj|j�|jD�]O}tjj|jd|���}tjj|�r|j||�s�	tjj|�r7tjd|���d|�d�}tj|dd��n8tjd	|���d20|�d|��}tj|dd��	tjj|jd|�d��}|dk(r�tjj|d�}tjj|d�}tjj|�r8tjj|�rt!j"|||g�nt%d|����|dk(r8tjj|d�}	t!j"||	g�nt!j"||g�t!||�|j&|<tjd|�d����Ry#tj$r&}tjd|�d
|���Yd}~���d}~wwxYw#t($r/}tjd|�d
t+|����Yd}~���d}~wwxYw#t($r+}tjdt+|����Yd}~yd}~wwxYw)Nz)%(asctime)s - %(levelname)s - %(message)s)�level�formatztree-sitter-z!Updating existing repository for zcd z && git pullT)�shell�checkzCloning repository for z5git clone https://github.com/tree-sitter/tree-sitter-� z&Failed to clone/update repository for z	. Error: zbuild/z.sor!r(z)TypeScript or TSX directory not found in r"zSuccessfully built and loaded z parserz!Failed to build or load language z9An unexpected error occurred during parser installation: )�logging�basicConfig�INFO�os�path�exists�	CACHE_DIR�makedirsr-r�_is_repo_valid�info�21subprocess�run�CalledProcessError�errorr�
build_library�FileNotFoundErrorr/�	Exceptionr+)22r1�language�	repo_path�update_command�
clone_command�e�23build_path�ts_dir�tsx_dir�php_dirs24          rr0zCodeParser._install_parsers(s������'�,�,�7b�c�.	`��7�7�>�>�$�.�.�1����D�N�N�+� �/�/�&
c���G�G�L�L����<��z�9R�S�	��w�w�~�~�i�0��8K�8K�I�W_�8`�!��7�7�>�>�)�4�#�L�L�+L�X�J�)W�X�/2�9�+�\�-J�N�&�N�N�>��T�R�#�L�L�+B�8�*�)M�N�.c�dl�cm�mn�ox�ny�,z�M�&�N�N�=��D�Q�25c�!#�����d�n�n��x�j�PS�>T�!U�J� �<�/�!#�����i��!F��"$�'�'�,�,�y�%�"@���7�7�>�>�&�1�b�g�g�n�n�W�6M�$�2�2�:���?P�Q�"3�6_�`i�_j�4k�"l�l��5�(�"$�'�'�,�,�y�%�"@�� �.�.�z�G�9�E� �.�.�z�I�;�G�/7�26�H�/M�D�N�N�8�,��L�L�#A�(��7�!S�T�I&
c��&�8�8�!��
�
�(N�x�j�Xa�bc�ad�&e�f� ��!��.!�c��M�M�$E�h�Z�y�Y\�]^�Y_�X`�"a�b�b��c���	`��M�M�U�VY�Z[�V\�U]�^�_�_��	`�sh�B6M�BK�-EL�M�L�"L�=M�L�M�	M�$L>�8M�>M�M�	M:�!M5�5M:rLrKrc��|dk(r�tjjtjj|ddd��xr?tjjtjj|ddd��S|dk(r?tjjtjj|ddd��Stjjtjj|dd��S)z1Check if the repository contains necessary files.r!�srczparser.cr(r")r=r>r?r)r1rLrKs   rrBzCodeParser._is_repo_valid[s����|�#��G�G�N�N�2�7�7�<�<�	�<��PZ�#[�\�W��W�W�^�^�B�G�G�L�L��E�5�*�$U�V�
X�
��
��7�7�>�>�"�'�'�,�,�y�%��27�"S�T�T��7�7�>�>�"�'�'�,�,�y�%��"L�M�M�r28�file_extensionc�L�|jj|�}|�td|���y|jj|�}|�td�yt	�}|j|�|j
t|d��}|�td�y|jS)N�Unsupported file type: �Language parser not found�utf8zFailed to parse the code)	r,r.�printr/r�set_language�parse�bytes�	root_node)r1r29rW�
language_namerK�parser�trees       r�30parse_codezCodeParser.parse_codees����3�3�7�7��G�
�� ��+�N�+;�<�=���>�>�%�%�m�4�����-�.��������H�%��|�|�E�$��/�0���<��,�-���~�~�rV�nodec��|j|�}g}|j|j�vr |j|||jf�|jD]#}|j|j
||���%|S�N)�_get_node_types_of_interest�type�keys�append�children�extend�extract_points_of_interest)r1rerW�node_types_of_interest�points_of_interest�childs      rrnz%CodeParser.extract_points_of_interestzs���!%�!A�!A�.�!Q�����9�9�.�3�3�5�5��%�%�t�-C�D�I�I�-N�&O�P��]�]�	^�E��%�%�d�&E�&E�e�^�&\�]�	^�"�!rVc���ddddd�ddd�ddddd	d31d�ddddd	d32dd
d�ddddddd�dddddd�d�}||j�vr||S|dk(r|dS|dk(r|dStd��)N�Import�Export�Class�Function)�import_statement�export_statement�class_definition�function_definition�TagzMedia Query)�tag_namez@mediazArrow Function�Block)rwrx�class_declaration�function_declaration�arrow_function�statement_block�	Interfacez33Type Alias)rwrxr~rr�r��interface_declaration�type_alias_declaration�	Namespace�Method�Trait)�namespace_definitionr~�method_declarationrzr��trait_declaration�ModulezSingleton ClasszBegin Block)�class�method�module�singleton_class�begin�r$r r%r'r"r)r&r%r(r'�Unsupported file type�rj�34ValueError�r1rW�35node_typess   rrhz&CodeParser._get_node_types_of_interest�s���%-�$,�$+�'1�	�"�'��36%-�$,�%,�(2�"2�#*�
�%-�$,�%,�(2�"2�#*�)4�*6�	�)4�%,�&.�'1�)4�%,�
�!�"�"�#4�&��K,37�38�\�Z�_�_�.�.��n�-�-�
�u�
$��d�#�#�
�u�
$��d�#�#��4�5�5rVc��ddd�ddiddd�ddd�ddd�ddid�}||j�vr||S|dk(r|d	Std39��)N�Comment�	Decorator)�comment�	decoratorr��	Attribute)r��	attributer�r&r%r�r�r�s   r�_get_nodes_for_commentsz"CodeParser._get_nodes_for_comments�s���%�(��40�9��%�(��41%�(��42%�(��43�9��)44�45�2�Z�_�_�.�.��n�-�-�
�u�
$��d�#�#��4�5�5rVc���|j|�}g}|j|vr |j|||jf�|jD]#}|j	|j||���%|Srg)r�rirkrlrm�extract_comments)r1rerWro�commentsrqs      rr�zCodeParser.extract_comments�su��!%�!=�!=�n�!M�����9�9�.�.��O�O�T�#9�$�)�)�#D�E�F��]�]�	J�E��O�O�D�1�1�%��H�I�	J��rVc�$�|jj|�}|�td��|jj|�}|�td��t	�}|j|�|j
t|d��}|j}|j||�}i}	|D]9\}46}|47jd}||	vrg|	|<||	|vs�&|	|j|��;g}
|	j�D]\}}|
j|��|
S�Nr�rZr[r)r,r.r�r/rr]r^r_r`rn�start_pointrk�itemsrm)r1r48rWrarKrbrcr`rp�"line_numbers_with_type_of_interestre�type_of_interest�49start_line�lines_of_interest�_�line_numberss                r� get_lines_for_points_of_interestz+CodeParser.get_lines_for_points_of_interest�s5���3�3�7�7��G�
�� ��4�5�5��>�>�%�%�m�4�����8�9�9�������H�%��|�|�E�$��/�0���N�N�	�!�<�<�Y��W��-/�*�&8�	X�"�D�"��)�)�!�,�J��'I�I�GI�2�3C�D��!C�DT�!U�U�2�3C�D�K�K�J�W�
	X���A�G�G�I�	3�O�A�|��$�$�\�2�	3�!� rVc�$�|jj|�}|�td��|jj|�}|�td��t	�}|j|�|j
t|d��}|j}|j||�}i}	|D]9\}50}|51jd}||	vrg|	|<||	|vs�&|	|j|��;g}
|	j�D]\}}|
j|��|
Sr�)r,r.r�r/rr]r^r_r`r�r�rkr�rm)r1r52rWrarKrbrcr`r��line_numbers_with_commentsrer�r�r�r�r�s                r�get_lines_for_commentsz!CodeParser.get_lines_for_comments
s4���3�3�7�7��G�
�� ��4�5�5��>�>�%�%�m�4�����8�9�9�������H�%��|�|�E�$��/�0���N�N�	��(�(��N�C��%'�"�&.�	P�"�D�"��)�)�!�,�J��'A�A�?A�*�+;�<��!;�<L�!M�M�*�+;�<�C�C�J�O�
	P���9�?�?�A�	3�O�A�|��$�$�\�2�	3�!� rVc	���|jj|�}|�td|���y|jj|�}|�td�yt	�}|j|�|j
t|d��}|j}|j|�}|jd�}	|j�D]0\}53}|	|54dz55}td|56�ddj|��d	|����2y)57NrYrZr[r
rzline rz, z	 | Code: )
r,r.r\r/rr]r^r_r`�map_line_to_node_typerr�r)
r1r58rWrarKrbrcr`�line_to_node_typer�line_numr��line_contents
             r�print_all_line_typeszCodeParser.print_all_line_types.s����3�3�7�7��G�
�� ��+�N�+;�<�=���>�>�%�%�m�4�����-�.��������H�%��|�|�E�$��/�0���N�N�	� �6�6�y�A���Z�Z��%�59�$5�$;�$;�$=�	V� �H�j�%�h��l�3�L��E�(��2�d�i�i�60�&;�%<�I�l�^�T�U�	VrVc���|�i}|jddz}||vrg||<||j|j�|jD]}|j	|||dz��|S)Nrr)r�rkrirlr�)r1rer��depthr�rqs      rr�z CodeParser.map_line_to_node_typeGs����$� "���%�%�a�(�1�,�61��.�.�,.��j�)��*�%�,�,�T�Y�Y�7��]�]�	L�E��&�&�u�.?����K�	L�!� rVrg)Nr)�__name__�62__module__�__qualname__r=r>�63expanduserr@rrr+r3r0�boolrBr	rdrrnrrhr�r��intr�r�r�r��rVrrrsd�����"�"�#9�:�I� ��d�D��I�s�.B�(C� �*1`�fN��N�s�N�t�N��s��C��E�$��*�<M��*64"�t�65"�S�66"�T�RW�X\�^a�Xa�Rb�Mc�67"�66�#�66�$�s�C�x�.�66�r6�c�6�d�3��8�n�6�B68�T�69�3�70�4��d�TW�i�HX�CY�71�!�S�!�#�!�RV�WZ�R[�!�B!�3�!��!��S�	�!�BV��V�c�V�2!rVr)r=rD�typingrrrr�tree_sitterrrr	r:r+rrr�rVr�<module>r�s<��	��+�+�.�.����s��s��F!�F!rV