Team Ai
Apppublic

sameersyed/Defence_FrameWork

sourceHugging Faceupdated 8mo agoView on Hugging Face
0likes
dataset_analyzer.cpython-313.pyc163 linesDownload Raw Back to __pycache__
1�

2�ia:���SSKrSSKrSSKr\R3"S5 SSKJr SSK	r4SSKJr SSK
r
SSKr"SS5rg)�N�Agg)�BytesIOc��\rSrSrSrSrSrSrSrSr	Sr5S	rS6rSr
SrS
rSrSrSrSrSrSrSrSrg)�DatasetAnalyzer�c��SSSSS.Ulg)Ngffffff�?g333333�?��?g�������?)�	poisoning�	imbalance�outliers�7duplicates��risk_thresholds)�selfs �+D:\Security_app\backend\dataset_analyzer.py�__init__�DatasetAnalyzer.__init__s������	 8���c	��URU5URU5URU5URU5UR	U5URU5UR
U5S.nURUS5US'U$)z'Comprehensive dataset security analysis)�9basic_info�risks�class_distributionrr
�missing_values�10statisticsr�overall_risk)�_get_basic_info�
_detect_risks�_analyze_classes�_detect_outliers�_check_duplicates�_analyze_missing�_get_statistics�_calculate_risk_score)r�df�resultss   r�analyze_dataset�DatasetAnalyzer.analyze_datasets����.�.�r�2��'�'��+�"&�"7�"7��";��-�-�b�1��0�0��4�"�3�3�B�7��.�.�r�2�11��#'�"<�"<�W�W�=M�"N�����rc	�.�[U5[UR5[UR[R/S9R5[URS/S9R5URSS9R
5S-SS3S.$)	N��include�objectT)�deepi�.2fz KB)�
total_samples�total_features�numeric_features�categorical_features�memory_usage)�len�columns�
select_dtypes�np�numberr2�sum�rr$s  rr�DatasetAnalyzer._get_basic_info%s��� ��W�!�"�*�*�o� #�B�$4�$4�b�i�i�[�$4�$I�$Q�$Q� R�$'��(8�(8�(��(8�(L�(T�(T�$U�!���D��9�=�=�?�$�F�s�K�3�O�12�	13rc��/n[UR5S:�a�URSS2S4nURS;aaUR	5n[U5S:�aBUR5UR
5-nUS:aURSSSS	US14S3S.5 UR5R5[U5-nUS
:�aURSSSUS-SS3S.5 UR5R5R5[U5[UR5--nUS:�aURSSSUS-SS3S.5 UR[R/S9RnSn	UHZn15XRS5nXRS5nX�-16n
XUSU
--17:XUSU
--:�-R5nX�-
n	M\ [U5S:�aU	[U5[U5--OSnUS
:�aURSSSUS-SS3S.5 U(dURSSS S!S.5 U$)"Nr������r+�int64�g�������?zClass Imbalance�HIGH�Uz(Severe class imbalance detected (ratio: z.3f�))�type�severity�score�descriptiong�������?zDuplicate Samples�MEDIUM�<�dz.1fz% duplicate samples foundzMissing Data�7z% missing valuesr)��?��?��?zOutliers Detected�2z% outlier values detectedzNo Major Risks�LOW�18zDataset appears clean)r3r4�iloc�dtype�value_counts�min�max�append�19duplicatedr8�isnullr5r6r7�quantile)rr$r�last_col�class_counts�ratio�	dup_ratio�
missing_ratio�numeric_cols�
outlier_count�col�Q1�Q3�IQRr�
outlier_ratios                rr�DatasetAnalyzer._detect_risks.sz�����r�z�z�?�Q���w�w�q�"�u�~�H��~�~�!4�4�'�4�4�6���|�$�q�(�(�,�,�.��1A�1A�1C�C�E��s�{����$5�(.�%'�-U�V[�\_�U`�`a�+b�	&���M�M�O�'�'�)�C��G�3�	��t���L�L�+�$��"+�C�-��!4�4M�N�	�
��	�	����)�-�-�/�3�r�7�S����_�3L�M�
��3���L�L�&�$��"/��"3�C�!8�8H�I�	�
��'�'�����'�<�D�D���
��C���!�!�$�'�B���!�!�$�'�B��'�C���B��s��N�3���2��c�	�>�8R�S�X�X�Z�H��%�M� �JM�\�IZ�]^�I^�
��R��3�|�3D�)D�E�de�
��4���L�L�+�$��"/��"3�C�!8�8Q�R�	�
���L�L�(�!��6�	�
��rc��[UR5S:Xa0$URSS2S4nURS;aUR	5R5$0$)Nrr<r=)r3r4rQrRrS�to_dict)rr$rZs   rr� DatasetAnalyzer._analyze_classesqsR���r�z�z�?�a���I��7�7�1�b�5�>���>�>�0�0��(�(�*�2�2�4�4��	rc�2�UR[R/S9Rn0nUSSHcnXR	S5nXR	S5nXe-20nXUSU--21:XUSU--:�-n[UR
55X4'Me U$)Nr)�rKrLrM)r5r6r7r4rY�intr8)	rr$r_rrarbrcrd�outlier_masks	         rr� DatasetAnalyzer._detect_outlierszs����'�'�����'�<�D�D�������#�C���!�!�$�'�B���!�!�$�'�B��'�C��G�r�C�#�I�~�6�2�7�b�3�QT�9�n�;U�V�L��� 0� 0� 2�3�H�M�$��rc���[UR5R55[UR5R5[	U5-S-S5S.$)NrI�)�total_duplicates�22percentage)rlrWr8�roundr3r9s  rr �!DatasetAnalyzer._check_duplicates�sJ�� #�B�M�M�O�$7�$7�$9� :���
�
�� 3� 3� 5��B�� ?�#� E�q�I�23�	24rc��UR5R5nUR5VVs0sHup4US:�dM
U[U5_M snn$s snnf)Nr)rXr8�itemsrl)rr$�missingra�counts     rr!� DatasetAnalyzer._analyze_missing�sD���)�)�+�/�/�#��29�-�-�/�O�/�J�C�U�Q�Y���S��Z��/�O�O��Os�A�Ac�z�UR[R/S9n[UR5S:Xa0$UR5R
5nUR5VVVVs0sH5upEXER5VVs0sHupgU[US5_M snn_M7 snnnn$s snnfs snnnnf)Nr)rrp)	r5r6r7r3r4�describerhrvrs)rr$�25numeric_df�stats�k�vals�stat�vs        rr"�DatasetAnalyzer._get_statistics�s����%�%�r�y�y�k�%�:�26��z�!�!�"�a�'��I��#�#�%�-�-�/��RW�R]�R]�R_�`�R_�w�q��Z�Z�\�B�\�'�$�D�%��1�+�%�\�B�B�R_�`�`��B��`s�-B527�B/�!B528�/B529c�N�U(dg[SU55n[US5$)Nrc3�*# �UH	oSv� M g7f)rEN�)�.0�risks  r�	<genexpr>�8DatasetAnalyzer._calculate_risk_score.<locals>.<genexpr>�s���:�E�D�w�-�E�s�rI)r8rT)rr�total_scores   rr#�%DatasetAnalyzer._calculate_risk_score�s%�����:�E�:�:���;��$�$rc��0nURUS5US'US(aURUS5US'US(aURU5US'URU5US'UR	US5US	'UR[R/S309n[UR5S:�aURU5US'U$)
zGenerate visualization chartsr�risk_pier�	class_barr�missing_heatmap�outlier_boxr�31risk_gauger)r?�correlation)�_create_risk_pie�_create_class_bar�_create_missing_heatmap�_create_outlier_box�_create_risk_gauger5r6r7r3r4�_create_correlation_heatmap)rr$�analysis�chartsr|s     r�generate_charts�DatasetAnalyzer.generate_charts�s�����"�2�2�8�G�3D�E��z���(�)�"&�"8�"8��BV�9W�"X�F�;���$�%�(,�(D�(D�R�(H�F�$�%�!%� 8� 8�� <��}�� $�6�6�x��7O�P��|���%�%�r�y�y�k�%�:�32��z�!�!�"�Q�&�$(�$D�$D�Z�$P�F�=�!��
rc�R�[R"SS9up#UVs/sHoDSPM	 nnUVs/sHoDSPM	 nnUVs/sHoDSS:XaSOUSS:XaS	OS33PM nnURXeUSSS
9 URSSSS9 UR	U5$s snfs snfs snf)N)����figsizerCrErDr@�#ff6b6brG�#ffa500�#51cf66z%1.1f%%�Z)�labels�colors�autopct�34startanglezRisk Distribution��bold��fontsize�35fontweight)�plt�subplots�pie�	set_title�_fig_to_base64)rr�fig�ax�rr��sizesr�s        rr�� DatasetAnalyzer._create_risk_pie�s����,�,�v�.���%*�+�U��F�)�U��+�%*�+�U��7��U��+�|A�B�|A�vw��}��6�)��:��Zb�Ib�I�hq�q�|A��B�36���u�F�I�RT��U�37���(�2�&��I��"�"�3�'�'��,��+��Bs�B�B�!B$c	��[R"SS9up#[UR55SSnUVs/sHoQUPM	 nnUR	[[
U55USS9nURSSS9 URS	SS9 URS38SSS
9 UR[[
U555 URUSSS9 UHRnUR5n	URUR5UR5S--U	[!U	5SSS9 MT [R""5 UR%U5$s snf)N�rPr�r�rPz#667eea��color�Class��r�zSample CountzClass Distributionr�r�r��-�right��rotation�hag@�center�bottom)r��va)r�r��list�keys�bar�ranger3�39set_xlabel�40set_ylabelr��41set_xticks�set_xticklabels�42get_height�text�get_x�	get_widthrl�tight_layoutr�)43r�44class_distr�r��classes�c�counts�barsr��heights45          rr��!DatasetAnalyzer._create_class_bar�s.���,�,�w�/����z���(�)�#�2�.��)0�1��A�Q�-���1��v�v�e�C��L�)�6��v�C��46�
�
�g��
�+�47�
�
�n�r�
�2�48���)�B�6��J�49�
�
�e�C��L�)�*�50���7�R�G��<��C��^�^�%�F��G�G�C�I�I�K�#�-�-�/�"�"4�4�f��&�k�]��X�
�
?��51	�����"�"�3�'�'��!2s�E
c��[R"SS9up#UR5R5nXDS:�SSn[	U5S:�a�UR[
[	U55URSS9 UR[
[	U555 URUR5 URSSS	9 URS52SSS
9 [R"5 URU5$)Nr�r�rrPr�r�z
Missing Countr�r�zMissing Values by Featurer�r�r�)r�r�rXr8r3�barhr��values�53set_yticks�set_yticklabels�indexr�r�r�r�)rr$r�r�rws     rr��'DatasetAnalyzer._create_missing_heatmap�s����,�,�w�/����)�)�+�/�/�#���A�+�&�s��+���w�<�!���G�G�E�#�g�,�'����y�G�I��M�M�%��G��-�.����w�}�}�-��M�M�/�B�M�7��L�L�4�r�f�L�U������"�"�3�'�'rc�~�UR[R/S9RSSn[	U5S:�a�[54R"SS9up4XRUS9 URSSS	S559 URSSS
9 [56R"SSS9 [57R"5 URU5$g)Nr)rkrr�r�)r�zOutlier Detection (Box Plot)r�r�r��Valuer�r�r�r�r�)
r5r6r7r4r3r�r��boxplotr�r��xticksr�r�)rr$r_r�r�s     rr��#DatasetAnalyzer._create_outlier_box�s����'�'�����'�<�D�D�R�a�H���|��q� ��l�l�7�3�G�C���$�$��$�+��L�L�7�"�QW�L�X��M�M�'�B�M�/��J�J��w�/������&�&�s�+�+�rc58�Z�[R"SSS0S9up#[R"S[RS5n[R59"S5nUS:aSnS	nOUS60:aSnSnOS
nSnUR
XESSS9 UR
US[U5US[U5USS9 URSS5 UR/5 UR/5 UR[RS-SUSU3SSSSS9 URSSSSS9 URU5$)N)r���61projection�polar)r��62subplot_kwrrI�r�zLOW RISKrHr�zMEDIUM RISKr�z	HIGH RISK�	lightgray�)r��	linewidthrMrpr	z%63r��r�)r�r�r�r�zOverall Risk Scorer�)r�r��pad)r�r�r6�linspace�pi�ones�plotrl�set_ylimr�r�r�r�r�)r�64risk_scorer�r��thetar�r��labels        rr��"DatasetAnalyzer._create_risk_gauge�s(���,�,�v�<��:Q�R������A�r�u�u�c�*���G�G�C�L����?��E��E�
�"�_��E�!�E��E��E�65�����r��:�66����&�s�:��'��+;�C�67�O�)<�E�UW��X�68���A�s��69�
�
�b��70�
�
�b��71������a�����C��w�7�H���v�	�	/�72���)�B�6�r��R��"�"�3�'�'rc���[R"SS9up#UR5n[R"USSSSSSSS	0US739	 URSSS
S9 [R"5 URU5$)N)rPr�r�Tr-�coolwarmrr?�shrinkg�������?)�annot�fmt�cmapr��square�74linewidths�cbar_kwsr�zFeature Correlation Matrixr�r�r�)r�r��corr�sns�heatmapr�r�r�)rr|r�r�rs     rr��+DatasetAnalyzer._create_correlation_heatmapsr���,�,�w�/������ �����D��%�j���1��#��2�	O�75���1�B�6��R������"�"�3�'�'rc���[5nURUSSSS9 URS5 [R"UR55R
S5n[R"U5 SU3$)N�pngrI�tight)�format�dpi�bbox_inchesrzutf-8zdata:image/png;base64,)	r�savefig�seek�base64�	b64encode�read�decoder��close)rr��buf�76img_base64s    rr��DatasetAnalyzer._fig_to_base64 sa���i�����C��3�G��D�������%�%�c�h�h�j�1�8�8��A�77��	�	�#��'�78�|�4�4rc�.�S[RR5RS5SUSSSUSSSUSS	S79USSSUSS
SUSS:�aSOUSS:�aSOSSUSS3nUSH5nUSUSSUSR	5SUSSUSSUSS3-
nM7 US -
nUR5H;upVU(dMUS!UR
S"S#5R5S$US%US&3-
nM= US'-
nU$)(z!Generate downloadable HTML reportu�80<!DOCTYPE html>81<html>82<head>83    <title>Dataset Security Analysis Report</title>84    <style>85        body { font-family: Arial, sans-serif; margin: 40px; background: #f5f5f5; }86        .container { max-width: 1200px; margin: 0 auto; background: white; padding: 30px; border-radius: 10px; }87        h1 { color: #667eea; border-bottom: 3px solid #667eea; padding-bottom: 10px; }88        h2 { color: #333; margin-top: 30px; }89        .risk-high { color: #ff6b6b; font-weight: bold; }90        .risk-medium { color: #ffa500; font-weight: bold; }91        .risk-low { color: #51cf66; font-weight: bold; }92        .info-box { background: #f0f0f0; padding: 15px; border-radius: 5px; margin: 10px 0; }93        .chart { margin: 20px 0; text-align: center; }94        .chart img { max-width: 100%; border: 1px solid #ddd; border-radius: 5px; }95        table { width: 100%; border-collapse: collapse; margin: 20px 0; }96        th, td { padding: 12px; text-align: left; border-bottom: 1px solid #ddd; }97        th { background: #667eea; color: white; }98    </style>99</head>100<body>101    <div class="container">102        <h1>🔒 Dataset Security Analysis Report</h1>103        <p><strong>Generated:</strong> z%Y-%m-%d %H:%M:%Su�</p>104        105        <h2>📊 Basic Information</h2>106        <div class="info-box">107            <p><strong>Total Samples:</strong> rr.z5</p>108            <p><strong>Total Features:</strong> r/z7</p>109            <p><strong>Numeric Features:</strong> r0z;</p>110            <p><strong>Categorical Features:</strong> r1z3</p>111            <p><strong>Memory Usage:</strong> r2u�</p>112        </div>113        114        <h2>⚠️ Risk Assessment</h2>115        <div class="info-box">116            <p><strong>Overall Risk Score:</strong> <span class="risk-rrH�highr��medium�lowz">z�%</span></p>117        </div>118        119        <h3>Detected Risks:</h3>120        <table>121            <tr><th>Risk Type</th><th>Severity</th><th>Score</th><th>Description</th></tr>122rz&123            <tr>124                <td>rCz&</td>125                <td class="risk-rDz</td>126                <td>rErFz</td>127            </tr>128u@129        </table>130        131        <h2>📈 Visualizations</h2>132z-133        <div class="chart">134            <h3>�_� z</h3>135            <img src="z" alt="z">136        </div>137u�138        <h2>🛡️ Recommendations</h2>139        <ul>140            <li>Remove duplicate samples to prevent data poisoning</li>141            <li>Balance class distribution using SMOTE or undersampling</li>142            <li>Handle missing values with imputation or removal</li>143            <li>Detect and remove outliers using IQR method</li>144            <li>Validate data sources and apply sanitization</li>145            <li>Use adversarial training for robust models</li>146        </ul>147        148        <p style="margin-top: 40px; text-align: center; color: #888;">149            Generated by ShieldML - Deep Learning Security Analysis System150        </p>151    </div>152</body>153</html>154)�pd�	Timestamp�now�strftime�lowerrv�replace�title)rr�r��htmlr��155chart_name�156chart_datas       r�generate_report_html�$DatasetAnalyzer.generate_report_html(s��(�0)+���(8�(8�(:�(C�(C�DW�(X�'Y�Z0�19��0F��0W�/X�Y1�19�,�1G�HX�1Y�0Z�[3�3;�L�3I�J\�3]�2^�_7�7?��7M�Nd�7e�6f�g/�/7��/E�n�/U�.V�WG�157RZ�Zh�Qi�ln�Qn�v�AI�JX�AY�\^�A^�t|�di�Gj�jl�mu�vD�mE�lF�F�K+��Z�W�%�D����&�\�N�#!�!%�j�!1�!7�!7�!9� :�"�T�*�=M�<N�O��'�]�O�$��-�(�)�*��
�D�&�	
��	��'-�l�l�n�"�J��z����#�#�C��-�3�3�5�6�7�!�l�'�*��6����'5�	
��	��$�rrN)�__name__�158__module__�__qualname__�__firstlineno__rr&rrrrr r!r"r#r�r�r�r�r�r�r�r�r)�__static_attributes__r�rrrrsf��159��"160�A�F��161�P�a�%��8	(�(�*
(��(�8	(�5�Zrr)�pandasr�numpyr6�162matplotlib�use�matplotlib.pyplot�pyplotr��seabornr�iorr�jsonrr�rr�<module>r9s5�����163���u�����
��w�wr