Team Ai
Apppublic

aravagarwal/CodeCloakPII

sourceHugging Faceupdated 3y agoView on Hugging Face
0likes
rootanoninterface__pycache__treeutils.cpython-310.pyc
treeutils.cpython-310.pyc31 linesDownload Raw Back to __pycache__
1o

2=t�e��@s|ddlmZmZmZmZmZddlmZddlZ	dddd�Z3dd	�e4��D�Zdd5d�Z
gfdd
�Zdd�Zgfdd�ZdS)�)�parser�analyzer�6anonymizer�PYTHON_LANGUAGE�MAX_STRINGS_IN_SAMPLE)�deepcopyN�comment�string�name)z(comment) @commentz (string_content) @string_contentz(identifier) @identifiercCsi|]\}}||�qS�r)�.0�key�valuerr�Z/home/arava/update_feedback_data/text_pii_removal/anoninterface/anoninterface/treeutils.py�7<dictcomp>srcCs2|durt�|�d��}t�|�}|�|j�|fS)N�utf8)r�parse�encoder�query�captures�	root_node)�	node_text�querystr�treerrrr�get_capturess8rcCs�t|�d��}|�d�}|j}|j}|j|}	|j}9|j}|jd|jd|f}|d|jd�|�d�||jdd�}
|�|||	|10||�t�	|
|�|
�11d�fS)Nrr�)�lenr�12start_byte�end_byte�start_point�	end_point�13byte_range�editrr�decode)�node�
new_node_text�source_text�source_tree�captured_nodes�new_elem_bytes�source_bytes�node_bstart�node_estart�node_nestart�node_pbstart�node_pestart�
node_pnestart�new_source_textrrr�update_tree_and_sources1415.r2cCs�t||�\}}t|�}t|�D]/}t|�t||dj�d��}|||d�}	t||d|	||�\}16}|17}t|||�\}}q|}|S)Nrr)�anon_fn)rr�range�printr�textr#r2)�
solution_textr�18replace_fn�	filter_fnrr�number_of_edits�elem�old_text�new_text�new_tree�
redacted_textrrr� syntax_aware_replace_with_lambda"sr@c
sdd�|D�}|�d�}g}|D]�t|��\}}�fdd�|D�}|�|�qt|dd�d�}g}d}|D]3}	|	dj\}19}|20|krf|�|||21��d�df�|�||22|��d�t|	d	f�|}q6td23��dd�|D�}t	|�t24kr|t�d�|gkr�||fS|g|fS)
NcSsg|]}t|�qSr)�TREE_SITTER_QUERY_NAME_TO_QUERY)rrrrr�25<listcomp>2sz1visualize_tree_sitter_matches.<locals>.<listcomp>rcsg|]}|d�f�qS)rr�rr;�rrrrB7scSs|djdS)Nr)r!)�xrrr�<lambda>9sz/visualize_tree_sitter_matches.<locals>.<lambda>)r
rrz3Overlapping Query Spans Are Currently Not SupportedcSsg|]26}|dj�d��qS)rr)r6r#rCrrrrBDsz+Your code snippet has too many identifiers.)
rr�extend�sortedr!�appendr#�TREE_SITTER_QUERY_TO_QUERY_NAME�27ValueErrorrr�gradio�Warning)
r7�queries�unused�solution_text_bytes�total_capture_listrr�
labelled_text�read_loc�capture�
capture_start�capture_end�28spans_listrrDr�visualize_tree_sitter_matches1s*29$30rX)N)�
anoninterfacerrrrr�copyrrL�grrJ�itemsrArr2r@rXrrrr�<module>s�31