Team Ai
Apppublic

2008robocode-crypto/code-generation-system

sourceHugging Faceupdated 5mo agoView on Hugging Face
0likes
evaluator.cpython-314.pyc76 linesDownload Raw Back to __pycache__
1+
2A��iR)��T�Rt^RIt^RIt^RIHtHtHtHt^RIHt^RI	t	^RI3t4]	PP^]5PP]6PP]4RR44^RIHt^RIHt^RIHtHtHt!R	R74tRt]R8Xd8]!4R#R#)
z9Evaluation framework to measure system performance on test dataset.10Tracks success rate, retries, failure types, and latency.11N)�Any�Dict�List�Optional)�datetimez..�src)�Pipeline)�validate_config_executable)�get_test_dataset�get_real_prompts�get_edge_casesc�a�]tRt^toRtRV3RlRlltRV3RlRlltV3RlRltRV3RlR	lltV3R12lRlt	V3RlR
lt13V3RlRltRtRt
VtR#)�EvaluationFrameworkz'Comprehensive evaluation of the system.c� <�V^8�dQhRS[/#)��use_llm)�bool)�format�
__classdict__s"��BC:\Users\purav\Downloads\ai intern project\evaluation\evaluator.py�__annotate__� EvaluationFramework.__annotate__s���
14�
15��
16�c��\VR7Vn.VnR^R^R^R^R^RRR/R	/R17\P!4P4/	VnR#)�r�
total_prompts�18successful�failed�19executable�
total_retries�
total_latency��by_category�
failure_types�	timestampN)r�pipeline�resultsr�now�	isoformat�summary)�selfrs&&r�__init__�EvaluationFramework.__init__sZ�� ��1��
�����Q��!��a��!��Q��S��2��R������1�1�3�2021��rc�X<�V^8�dQhRS[S[S[3,RS[RS[S[S[3,/#)r�prompt�max_retries�return)r�strr�int)rrs"�rrr's5���<�<�d�3��8�n�<�3�<�t�TW�Y\�T\�~�<rc�b�VPRR4pVPRR4pVPRVPRR44pRVRVRVR	,R22RRRR
^RRR.R.R^/23p\P!4p\V4F�pV^,VR
&VPP	V4wr�V	'gVR,PR4KO\
V	4wr�RVR24&W�R&\\P!V	44VR&V'gQVR,PVPR.44VR,PVPR.44W�R&W�R&M	\P!4V,25VR&V# \d�p
\T
4pTR,PT4TPT4pY�PR,9d^TPR,T&TPR,T;;,^,
uu&Rp
?
EK�Rp
?
ii;i)zEvaluate a single prompt.�id�unknownr.��category�type�	prompt_id�prompt_summary:N�dN�successFr�retries�latencyr!�errors�warnings�config_sizezEmpty config generatedT�
execution_log�execution_reportr#N)�get�time�ranger%�generate�appendr	�len�json�dumps�extend�	Exceptionr1�_categorize_errorr))r*r.r/r9�prompt_textr7�result�26start_time�attempt�config�exec_log�
is_executable�exec_report�e�	error_msg�27error_types&&&             r�evaluate_prompt�#EvaluationFramework.evaluate_prompt's	���J�J�t�Y�/�	��j�j��2�.���:�:�j�&�*�*�V�Y�*G�H��
�����k�$�/��u��%��q��s��b����1�28���Y�Y�[�29��[�)�G� '�!��F�9�� 
?�#'�=�=�#9�#9�+�#F� ����8�$�+�+�,D�E��.H��-O�*�
�$(��y�!�'4�|�$�(+�D�J�J�v�,>�(?��}�%�$��:�&�-�-�k�o�o�j�"�.M�N��8�$�+�+�K�O�O�H�b�,I�J�+3��'�-8�)�*��3*�J!�I�I�K�*�4��y���
���
?���F�	��x� �'�'�	�2�"�3�3�I�>�30��\�\�/�%B�B�@A�D�L�L��1�*�=����_�-�j�9�Q�>�9�9��
?�s �=F�BF�H.�BH)�)H.c�&<�V^8�dQhRS[RS[/#)r�errorr0�r1)rrs"�rrres���#�#�s�#�s�#rc��VP4pRV9dR#RV9dR#RV9dR#RV9dR#R	V9gR31V9dR#R#)
zCategorize error type.rJ�32json_error�33validation�validation_error�field�field_error�api�	api_error�database�table�database_error�
unknown_error)�lower)r*r]�error_lowers&& rrN�%EvaluationFramework._categorize_erroresN���k�k�m���[� ��
�[�
(�%�
��
#� �
�k�
!��
�;�
&�'�[�*@�#�"rc�<<�V^8�dQhRS[RS[S[S[3,/#)r�dataset_sizer0)r1rr)rrs"�rrrvs#���3!�3!�3�3!�D��c��N�3!rc���VR8Xd\4\4,pM:VR8Xd\4pM(VR8Xd\4pM\4R\V4p\V4VPR&\R\V4R24\
V4EF�wr4\RV^,R	\V4R34VPRVPR442R
R7VPV4pVPPV4VR,'d,VPR;;,^,
uu&\R4M*VPR;;,^,
uu&\R4VR,'d VPR;;,^,
uu&VPR;;,VR,,
uu&VPR;;,VR,,
uu&VR,pW`PR,9dR^R^/VPR,V&VPR,V,R;;,^,
uu&VR,'gEK�VPR,V,R;;,^,
uu&EK�	VPR,^8�d3VPR,VPR,,^d,M^VPR&VPR,^8�d3VPR,VPR,,^d,M^VPR&VPR,^8�d,VPR,VPR,,M^VPR&VPR,^8�d,VPR,VPR,,M^VPR&VP4#) z$Run full evaluation on test dataset.�full�real�edgeNru35📊 Running evaluation on z prompts...z  [�/z] �namer4� )�endr<ru✓ru✗rrr=r r>r7r"�total�success_rate�executable_rate�avg_retries�avg_latency)rrr2rIr)�print�	enumeraterDrZr&rH�36get_report)r*ro�prompts�ir.rPr7s&&     r�run_evaluation�"EvaluationFramework.run_evaluationvs\���6�!�&�(�>�+;�;�G�
�V�
#�&�(�G�
�V�
#�$�&�G�&�(�);�#�l�*;�<�G�(+�G�����_�%�
�-�c�'�l�^�;�G�H�"�7�+�I�A��C��!��u�A�c�'�l�^�2�f�j�j�����D�AQ�.R�-S�T�Z]�^��)�)�&�1�F��L�L����'��i� � ����\�*�a�/�*��e�����X�&�!�+�&��e���l�#�#����\�*�a�/�*��L�L��)�V�I�->�>�)��L�L��)�V�I�->�>�)��j�)�H��|�|�M�:�:�9B�A�w�PQ�8R����]�+�H�5��L�L��'��1�'�:�a�?�:��i� � ����]�+�H�5�i�@�A�E�@�7,�<nr�my�my�{J�nK�NO�nO����\�(B�T�\�\�Ra�Eb�(b�fi�'i�UV����^�$�pt�p|�p|�~M�qN�QR�qR�4�<�<��+E����Ud�He�+e�il�*l�XY����&�'�gk�gs�gs�uD�hE�HI�hI�d�l�l�?�&C�d�l�l�Sb�Fc�&c�OP����]�#�gk�gs�gs�uD�hE�HI�hI�d�l�l�?�&C�d�l�l�Sb�Fc�&c�OP����]�#���� � rc�6<�V^8�dQhRS[S[S[3,/#�rr0�rr1r)rrs"�rrr�s���37�38�D��c��N�39rc�V�RVPRVPRVP4/#)zGenerate evaluation report.r)�detailed_results�
cost_analysis)r)r&�_calculate_cost_analysis)r*s&rr�EvaluationFramework.get_report�s.��
�t�|�|������T�:�:�<�40�	41rc�6<�V^8�dQhRS[S[S[3,/#r�r�)rrs"�rrr�s���42�43�$�s�C�x�.�44rc�$�VP'g/#VPUu.uFqR,'gKVNK	ppV'gRR/#\RV44\V4,p\RV44\V4,pRVR\V^4R^R	\	V^,4R45VP46R,R,VP47R
,R,,R^dV^48,,49RVP
4/#uupi)z!Analyze cost vs quality tradeoff.r<�notez$No successful generations to analyzec3�2"�TF
qR,x�K	R#5i)rAN���.0�rs& r�	<genexpr>�?EvaluationFramework._calculate_cost_analysis.<locals>.<genexpr>�s���K�8J�1�
�.�.�8J���c3�2"�TF
qR,x�K	R#5i)r>Nr�r�s& rr�r��s���C�0B�1�I�,�,�0B�r��avg_config_size_bytes�avg_generation_latency_seconds�estimated_api_calls_per_prompt�estimated_tokens_per_generation�
quality_scoreryg333333�?rzg�������?�efficiency_score�recommendation)r&�sumrI�roundr2r)�_get_recommendation)r*r��successful_configs�avg_config_sizer|s&    rr��,EvaluationFramework._calculate_cost_analysis�s����|�|�|��I�)-���F��A�9���a�a���F�!��B�C�C��K�8J�K�K�c�Rd�Ne�e���C�0B�C�C�c�J\�F]�]��
$�_�,�e�K��.C�,�a�-�s�?�Q�3F�/G��d�l�l�>�:�S�@�T�\�\�Rc�Ed�gj�Ej�k���{�R�'7� 8��d�6�6�8�50�	51��Gs52�D
�D
c� <�V^8�dQhRS[/#r�r^)rrs"�rrr�s���7�7�S�7rc���VPPR^4pVPPR^4pV^P8�d53V^K8�dR#V^<8�d54V^28�dR#V^(8�dR#R#)z$Get recommendation based on metrics.ryrzz Production-ready with monitoringz Ready for limited production usez"Needs refinement before productionz!Requires significant improvements)r)rD)r*ryrzs&  rr��'EvaluationFramework._get_recommendation�sa���|�|�'�'���:���,�,�*�*�+<�a�@���2��/�R�"7�5�
�R�
�O�r�$9�5�
�R�
�7�6rc��\R&4\R4\R%4VPp\R4\RVR,24\RVR,RVR,RVPR	^4R55
R24\RVR
,RVR,RVPR^4R56
R24\RVPR^4R
24\RVPR^4R
R24\R4VPR/4P4FZwr#VR,^8�dVR,VR,,^d,M^p\RVRVR,RVR,RVR
R2	4K\	\R4VPR4'd3VR,P4FwrV\RVRV24K	M\R4\R4VP	4pVP4Fwr�VR 8wgK\RVRV	24K!	\R!VPR"R#424\R'4R$#)(zPrint formatted report.u📊 EVALUATION REPORTu57📈 SUMMARY METRICS:z  Total Prompts Evaluated: rz  Successful Generations: rrtz (ryz.1fz%)z  Executable Configs: rrzz  Average Retries: r{z.2fz  Average Latency: r|�su58📁 RESULTS BY CATEGORY:r"rxr<z  z: z.0fu59❌ ERROR TYPES:r#z  None (all prompts succeeded!)u60💰 COST vs QUALITY ANALYSIS:r�u61✅ RECOMMENDATION: r��UnknownNzF======================================================================zG62======================================================================zG======================================================================63)r}r)rD�itemsr�)64r*r�r7�stats�success_pctrY�countr��key�values65&         r�print_report� EvaluationFramework.print_report�sE��
�m��
�&�'�
�f�
��L�L��
�'�)�
�+�A�o�,>�+?�@�A�
�*�1�\�?�*;�1�Q��=O�<P�PR�ST�SX�SX�Yg�ij�Sk�lo�Rp�pr�s�t�
�&�q���&7�q��?�9K�8L�B�q�u�u�Uf�hi�Oj�kn�No�oq�r�s�
�#�A�E�E�-��$;�C�#@�A�B�
�#�A�E�E�-��$;�C�#@��B�C�
�+�-� �u�u�]�B�7�=�=�?�O�H�GL�W�~�XY�GY�5��+�e�G�n�<�s�B�_`�K��B�x�j��5��#3�"4�A�e�G�n�5E�R��TW�GX�XZ�[�\� @�	�"�$��5�5��!�!�%&��%7�%=�%=�%?�!�66���:�,�b���0�1�&@�
�3�4�
�0�2��5�5�7�
�'�-�-�/�J�C��f�}���3�%�r�%��)�*�0�	�&�}�'8�'8�9I�9�'U�&V�W�X�
�m�r)r%r&r)N)T)�)rq)�__name__�67__module__�__qualname__�__firstlineno__�__doc__r+rZrNr�rr�r�r��__static_attributes__�__classdictcell__)rs@rrrsZ����1�
68�
69�<�<�|#�#�"3!�3!�j70�71�72�73�.7�7�!�!rrc�b�\RR7pVPRR7pVP4V#)z"Run the complete evaluation suite.Frrq)ro)rr�r�)�	evaluator�reports  r�run_evaluation_suiter��s0��#�E�2�I�
�
%�
%�6�
%�
:�F�
�����Mr�__main__)r�rJrE�typingrrrrr�sys�os�path�insert�join�dirname�__file__r%r�runtime_simulatorr	�test_datasetr74rrrr�r�r�rr�<module>r�s����75��,�,��76�	�������2�7�7�<�<������� 9�4��G�H��8�K�K�d�d�N��z����r