Team Ai
Apppublic

documentExtractionag051/ExtractDocument

sourceHugging Faceupdated 9mo agoView on Hugging Face
0likes
streamlit_app.cpython-313.pyc157 linesDownload Raw Back to __pycache__
1�

2��_i)T��r�SrSSKrSSKrSSKrSSKJrJrJ	r	J3r4 SSKrSSKJ
r
Jr SSKJrJr S\\S\\4SjrS	\S\\\44S5jrS\S\4SjrS
\S\S\4SjrS\\S\\\4S\S\4SjrSS\\S\S\S\S\S\\4SjjrS\\S\\S\S\S\\46SjrSr\S:Xa\"5 gg)aa7Streamlit Application for Digital PDF Table Extraction8 9This application provides a UI for:101. Uploading digital PDF documents112. Configuring table extraction parameters123. Viewing extracted tables13 14Entry point uses pdf_utils.py for text extraction and existing post-processing pipeline.15NO OCR - Only digital/searchable PDFs are supported in Phase-1.16�N)�Dict�List�Any�Optional)�extract_text_from_pdf_bytes�is_pdf_digital)�TextRect�Rect�17ocr_blocks�returnc�<�/nUH�nURS5S:XdM[USSUSSUSSUSSURSS	5URSS5URS18S5URSS
5S9nURU5 M� U$)zQ19Create TextRect objects from ocrBlocks.20Adapted from pp_utils.create_text_rects21�	blockType�LINE�geometry�x1�y1�x2�y2�22confidenceg�?�text��pageNum�)rrrrr�23block_typer�page_num)�getr	�append)r�24text_rects�elem�	text_rects    �eC:\Users\poona\OneDrive\Documents\nextlifyai\Document_Automation\Document_Extraction\streamlit_app.py�!create_text_rects_from_ocr_blocksr"s���25�J����8�8�K� �F�*� ��26�#�D�)��27�#�D�)��28�#�D�)��29�#�D�)��8�8�L�#�6��8�8�K��8��X�X�f�b�)����)�Q�/�	�I�
���i�(�����extracted_datac���0nURS/5HGnURS[U5S-5nURSS5URSS54X'MI U$)z0Extract page dimensions from extracted PDF data.�pagesrr�widthr�height)r�len)r$�page_dimensions�pagers    r!�extract_page_dimensionsr,2sc���O��"�"�7�B�/���8�8�I�s�?�';�a�'?�@��%)�X�X�g�q�%9�4�8�8�H�a�;P�$Q��!�0��r#rc�x�SRUR5R5R55$)zNormalize text for comparison.� )�join�lower�strip�split)rs r!�normalize_textr3;s)���8�8�D�J�J�L�&�&�(�.�.�0�1�1r#�	elem_text�header_textc�B�UR5nUR5nX#:XagUR5UR5:Xag[U5n[U5nXE:XagXT;agXE;agURU5(agURU5(agg)	zMulti-strategy header matching.)�exact�d)�case_insensitive�_)�30normalized�Z)�contains_header�P)�
contains_elem�K)�starts_with�F)�
header_starts�A)Nr)r1r0r3�31startswith)r4r5�32elem_clean�header_clean�	elem_norm�header_norms      r!�match_headerrJ@s������"�J��$�$�&�L��!������\�/�/�1�1�'��z�*�I� ��.�K���!���&���$����K�(�(�"����i�(�(�$�r#�	ocr_elems�column_headersr*c�^�U(a [SUR555OSnUVs/sH(oDRS:XdMURS:XdM&UPM* nn0n0nUR	5H�up�Sn33SnSnUH(n[URU	5up�X�:�dM"UnU
n34UnM* U35(dMBU(dMKURSURUURS.Xh'U	URR5U36UURSUR3S.Xx'M� UUS	.$s snf)37z/Detect table columns based on header positions.c3�*# �UH	upUv� M g7f�N�)�.0�w�hs   r!�	<genexpr>�!detect_columns.<locals>.<genexpr>ds���A�(@���!�(@�s�rrrN)rrrr�header_y�-)r5�matched_text�38match_typer�x_range)�
column_spaces�matched_headers)�max�valuesrr�itemsrJrrrrr1)rKrLr*�max_page_heightr�page1_linesr[r\�col_keyr5�39best_match�best_confidence�	best_elemrYrs               r!�detect_columnsrfbs2��ET�c�A��(>�(>�(@�A�A�Z[�O�%.�b�I�D���!�1C�4����[a�Ha�4�I�K�b��M��O� .� 4� 4� 6����40����	��D�%1�$�)�)�[�%I�"�J��+�",��'�41� �	� ��:�)�)��l�l���l�l�%�%�L�L�&�M�"� +� )��� 4� 4� 6�(�-�'�l�l�^�1�Y�\�\�N�;�(�O�$�)!7�:'�*����Ccs�D*�D*�D*�anchor_column_space�anchor_pattern�	stop_textc42���SSKnU(d/$U(aURU5OURS5n/nUH�nURS:waMURUR	SS5:aM7UR43UR-S-n	USn44USnX�-45S	-nX�-46U	s=::a47X�-::dMwO M{URR5n
URU
5(dM�URU
UURURURS48.5 M� URSS9 /n[U5H�unnU(a(UR5US
R5;a U$USnUR	US5unnUSnUS-[!U5:aXS-SU:XaXS-SnOUSUSUS-49S--nURUUS
UUSUS.5 M� U$)z(Identify table rows using anchor column.rNz.*rrV�rr��?)rrrrrc��USUS4$)NrrrP��xs r!�<lambda>�identify_rows.<locals>.<lambda>�s��Q�z�]�A�d�G�$<r#��keyrr)rrrrr)r�anchor_valuerrrr)�re�compilerrrrrrr1�matchrrr�sort�	enumerater0r))rKrgrhr*riru�pattern�anchor_linesr�
elem_center_x�col_x1�col_x2�	tolerancer�rows�i�anchorr�50page_width�page_heightrrs                      r!�
identify_rowsr��s�����	�,:�b�j�j��(��51�52�5�@Q�G��L����?�?�f�$���7�7�(�,�,�Z��;�;�����4�7�7�*�a�/�
�$�T�*��$�T�*���_��+�	����D�&�2D�D�D��9�9�?�?�$�D��}�}�T�"�"��#�#� � � $�
�
��'�'��'�'�%��%�6���<��=��D��|�,�	��6�����*�f�V�n�.B�.B�.D�D��(�K�%�*�%��"1�"5�"5�h��"G��53�K�
�D�\���q�5�3�|�$�$��!�e�)<�Z�)H�H�)T��!�e�$�T�*�B��������t��!<�� A�A�B���� �"�6�N�����
54�	�-�.�Kr#r�r[c�V�/nUGHnUSUSS.nUR5H�upx/n	UH�n55U56RS:waMU57RUS:waM*U58RU59R-S-nUSUs=::a	US::dO M]U60R61U62R-S-nUSn
US	nX�-63S64-nX�-65Us=::a66X�-::dM�O M�U	RU675 M� U	RSS9 S
RSU	55nUXg'M� URU5 GM" U$)z!Extract cell values for each row.rrt)rrtrrkrrrrrlc�2�URUR4$rO)rr)�es r!rp�$extract_table_rows.<locals>.<lambda>�s��1�4�4����,r#rrr.c3�T# �UHoRR5v� M  g7frO)rr1)rQr�s  r!rT�%extract_table_rows.<locals>.<genexpr>s��� D��A��������s�&()68r_rrrrrrrrxr/)rKr�r[r*�69table_data�row�row_datarb�	col_space�70cell_elemsr�
elem_center_yr|r}r~r�	cell_texts                 r!�extract_table_rowsr��sG���J����J����/�71��72#0�"5�"5�"7��G��J�!���?�?�f�,���=�=�C�73�O�3��"&���4�7�7�!2�a� 7�
��D�	�]�?�c�$�i�?��"&���4�7�7�!2�a� 7�
�"�4���"�4���#�_��3�	��%��L�&�:L�L�L��%�%�d�+�%"�*
�O�O� 6�O�7���� D�� D�D�I� )�H��9#8�<	���(�#�I�L�r#c
��[R"SSSS9 [R"S5 [R"S5 S[R;aS[RlS[R;aS[RlS	[R;aS[RlS74[R;aS[Rl[R [R"S5 [R"S5 [R"S
5 S[R;a)SSS.SSS.SSS.SSS.SSS./[Rl[R"SSS[[RR5S9n[[RR5U:ay[RRRS[[RR53SS.5 [[RR5U:aMy[[RR5U:�aQ[RRR!5 [[RR5U:�aMQ[#[RR5Hpup[R$"S US-S!US"3US#:S$9 [R&"S%US"S&U3S'9US"'[R&"S(US)S*U3S'9US)'SSS5 Mr [R("5 [R"S+5 [RRVs/sHo3S"PM	 nn[R*"S,UU(aS-OS-S.9n[R&"S/S0S1S29n[R("5 [R"S35 [R&"S4S5S6S29n[R,"S7S8S9S:S8S;S<9nSSS5 [R."SS/5up�U	 [R"S=5 [R0"S>S?/S@SA9nUGbP[R2"SBUR435 [R6"SCSDSE9(Ga[R8"SF5 UR;5n[=UUR4W5n
U
R?SG5(d[R@"SH5 O�U
[Rl[CU
5[Rl[EU
R?SG/55[Rl[R2"SI[U
R?SG/55SJU
SKSLSM35 SSS5 [RR75(Ga[R"SO5 [RR76R?SK05n[R."S#5unnnURKSPUR?SLS-55 URKSQ[[RR77R?SG/555 URKSR[[RR=(d /Vs/sHo�RLSS:XdMUPM sn55 SSS5 U78 [R"ST5 [RR79(Ga`[RR(Ga@[RRVs0sHnUS)(dMUS"US)_M nn[R6"SUSDSE9(Ga�[R8"SV5 [O[RRU[RR5nUSWnUSXn[R"SY5 U(aa[PRR"URU5VVs/sHunnUUSZUS[US\S]3S^.PM snn5n[RV"US_S`9 O[RX"Sa5 UR?W5n[[[RRUW[RRW5n\"Sb[U5Sc35 U(a[U(aT[_[RRUU[RR5nU[RlO/[RlSSS5 OO[R\"Se5 SSS5 [R("5 [RR(Ga	[R"Sf5 [PRR"[RR5n[R."Sg5unn URKSh[U55 U RKSi[UR.55 [RV"US_SjSk9 [R"Sl5 [R."S#5un!n"n#U! URgSmSn9n$[Rh"SoU$SpSqSrSs9 SSS5 U" [jRl"[RRSgSt9n%[Rh"SuU%SvSwSxSs9 SSS5 U# [RR80R?SK05[RR81R?Sy/5[RRSz.n&[Rh"S{[jRl"U&SgSt9S|SwS}Ss9 SSS5 [R$"S~SmS$9 [RR82(a�[RR83R?Sy/5H�n'U'R?SS5n([R"S�U(35 [RR84R?SG/5V)s/sH3n)U)R?S5U(:XdMU)R?S�5SS:XdM1U)PM5 n*n)[oU*S�Ss9H)n+[Rp"S�U+S�S�S�S�U+S�35 M+ M� SSS5 g!,(df   G85M�=fs snf!,(df   G	N9=f![Fa-n[R@"SN[IU535 SnAGN�SnAff=f!,(df   GN�=fs snf!,(df   GN�=fs snfs snnf![FaUn[R@"Sd[IU535 S-SK0n[Rb"URe55 SnAGN�SnAff=f!,(df   GN�=f!,(df   GN�=f!,(df   GN[=f!,(df   GN=f!,(df   GN�=fs sn)f!,(df   g=f)�NzPDF Table Extractoru📄�wide)�86page_title�	page_icon�layoutu 📄 Digital PDF Table ExtractorzJExtract tables from digital/searchable PDFs. **No OCR required** - Phase 1r$rKr*r�u⚙️ Table ConfigurationzColumn Headersz-Define the columns to extract from the table:�columns_config�
material_codez
Material Code)�87field_namer5�description�Description�quantity�Qty�88unit_pricez89Unit Price�total_price�TotalzNumber of Columnsr�)�	min_value�	max_value�value�column_rzColumn z: r��)�expandedz90Field Name�field_)r�rszHeader Text (to match)r5�header_zAnchor Column Settingsz!Anchor Column (for row detection)r)�options�indexzAnchor Pattern (Regex)z	^\d{6,8}$z3Regular expression to validate anchor column values)r��helpzAdvanced Optionsz	Stop Textz
Total Amount:zText that marks end of tablezLine Merge Thresholdg{�G�z�?g�������?g���Q��?z<Gap threshold for merging words into lines (% of page width))r�r�r��stepr�u📤 Upload PDFzChoose a digital PDF file�pdfzIOnly digital/searchable PDFs are supported. Scanned images will not work.)�typer�z91Uploaded: u🔍 Extract Text from PDF�primary)r�zExtracting text from PDF...�	ocrBlocksug❌ No text found in PDF. This might be a scanned document. Only digital PDFs are supported in Phase-1.u✅ Extracted z text blocks from �metadata�
numberOfPagesz pagesu❌ Error extracting PDF: u📊 Extraction Summary�PageszText Blocksz
LINE Elementsru📋 Extract Tablesu🔄 Extract Tablesz.Detecting columns and extracting table data...r[r\u🎯 Detected ColumnsrXrYr�%)�FieldzMatched Textz92Match Type�93ConfidenceT)�use_container_widthz:No columns detected. Check your header text configuration.zFound z table rowsu❌ Error extracting tables: u#👆 Upload and extract a PDF firstu📊 Extracted Table Datarkz94Total Rowsz
Total Columnsi�)r�r(u
📥 DownloadF)r�zDownload CSVzextracted_table.csvztext/csvzdownload-csvrr)�indentz
Download JSONzextracted_table.jsonzapplication/jsonz
download-jsonr&)r�r&r�zDownload Full Reportzfull_extraction_report.jsonz
download-fullu$🔍 View Raw Extracted Text (Debug)rzPage rc�"�USSUSS4$)NrrrrPrns r!rp�main.<locals>.<lambda>*s��a�95�m�D�>Q�ST�U_�S`�ae�Sf�=gr#�[rr�4dz] r)9�st�set_page_config�title�markdown�
session_stater$rKr*r��sidebar�header�	subheaderr��number_inputr)r�popry�expander�96text_input�divider�	selectbox�slider�columns�
file_uploader�success�name�button�spinner�readrr�errorr,r"�	Exception�str�metricrrf�pd�	DataFramer_�	dataframe�warningr��infor��	traceback�code�97format_exc�to_csv�download_button�json�dumps�sortedr),�num_columnsr��98col_config�c�anchor_options�
anchor_columnrhri�line_merge_threshold�col1�col2�
uploaded_file�	pdf_bytesr$r�r��col_a�col_b�col_crL�
column_resultr[r\�k�v�	header_df�anchor_col_spacer�r�r��df�col_m1�col_m2�col_d1�col_d2�col_d3�csv�json_str�	full_datar+rr�99page_lines�lines,                                            r!�mainrs������(�����H�H�
/�0��K�K�\�]��r�/�/�/�*.����'��"�*�*�*�%)����"��� 0� 0�0�+/����(��2�+�+�+�&*����#�100���101�	�	�.�/�102���%�&�103���C�D��2�#3�#3�3�.��O�,�]�K�)�%�@�+�L�I�,�W�E�/�B���+��o�o�&9�Q�RT�,/��0@�0@�0O�0O�,P�R���"�"�"�1�1�2�[�@����+�+�2�2�G�C�PR�P`�P`�Po�Po�Lp�Kq�Br�DF�4G�
H��"�"�"�1�1�2�[�@��"�"�"�1�1�2�[�@����+�+�/�/�1��"�"�"�1�1�2�[�@�'�r�'7�'7�'F�'F�G�M�A����w�q��s�e�2�j��.F�-G�H�ST�WX�SX�Y�+-�=�=� �104�<�(@��q�c�l�,�105�<�(�-/�M�M�,�J�}�4M�U\�]^�\_�S`�-�106�=�)�	Z�Y�H�	�107�108��109���-�.�35�3C�3C�3R�3R�S�3R�a�L�/�3R��S����/�"�%�!�1�110�
����$��F�111��	�112�113��114���'�(��M�M��!�/�115�	� "�y�y�"�����O�
 116��{117�N���Q��F�#�J�D�	
�118�	�	�#�$��(�(�'���\�119�
��$��J�J��M�$6�$6�#7�8�9��y�y�5�I�F�F��Z�Z� =�>�H�$1�$6�$6�$8�	�*E�%�)�.�.�0�*�� .�1�1�+�>�>��H�H�&O�P�>L�B�,�,�;�?V�We�?f�B�,�,�<�9Z� .� 2� 2�;�� C�:�B�,�,�6��J�J���N�<N�<N�{�\^�<_�8`�7a�as�uC�DN�uO�P_�u`�ta�ag�(h�i�+?�6���*�*�*��L�L�2�3��'�'�6�6�:�:�:�r�J�H�"$�*�*�Q�-��E�5�%��L�L��(�,�,���"B�C��L�L���B�,<�,<�,K�,K�,O�,O�P[�]_�,`�(a�b��L�L��#�"�:J�:J�:T�:T�:Z�XZ�:Z�.u�:Z�Q�^j�^j�nt�^t�q�:Z�.u�*v�w�a120�d121�122�	�	�'�(�
���*�*�*�r�/?�/?�/I�/I�/I��)�)�8�8��8�A��]�#�2��,���=�!1�1�8�
���y�y�.�Y�?�?��Z�Z� P�Q�68�(6��,�,�6�6�*��,�,�<�<�)�
�)6�o�(F�
�*7�8I�*J�����%<�=�*�(*���-<�,A�,A�,C�6�-D�D�A�q�./�45�n�4E�23�L�/�56�|�_�4E�Q�2G�	!"�-D�6�)�I��L�L���M��J�J�'c�d�,9�+<�+<�]�+K�(�,��,�,�6�6�,�*��,�,�<�<�%� �����&��T���;� ?�@� �M�);� "� 0� 0� :� :� $� -� "� 0� 0� @� @�	*�J�;E�B�,�,�7�:<�B�,�,�7�eR�Q�@�t
�G�G�9�:�K123�P�J�J�L�	���"�"�"�124�	�	�-�.��\�\�"�*�*�5�5�
6�����A������
�
�l�C��G�,��
�
�o�s�2�:�:��7�	���R�T�#�>�	���_�%�!#���A������
��)�)�%�)�(�C������%��"�
����z�z�"�"2�"2�"=�"=�a�H�H������&�"�#�
����,�,�;�;�?�?�125�B�O��)�)�8�8�<�<�W�b�I� �.�.�9�9��I�126
���&��127�128�9�Q�/�-�"�#�
�� 129���;�e�	L�
���*�*��(�(�7�7�;�;�G�R�H���8�8�I�q�1�����u�X�J�/�0�&(�%5�%5�%D�%D�%H�%H��VX�%Y��%Y�T��x�x�	�*�h�6��;?�8�8�K�;P�TZ�;Z��%Y���130#�:�3g�h�D��G�G�a��Z� 0�� 6�r�:�"�T�&�\�N�K�L�i�I�131M�	L�Z�Y��T�O132���\%�H����#=�c�!�f�X�!F�G�G��H��/?�>��D/v�a133���n��,6��J%�8����#@��Q���!I�J�(����	� 4� 4� 6�7�7��8��iR�Q��134���x�V���V���V��.�135M�	L�s��7Eu6�A6u6�Au6�?u�Au6�%u1�3Bu6�(Bw�.w�0Av�w�Bv�!w�"Dw�?w�w�w�1A3y*�$w+�6
w+�3y*�6y�8Bw6�? w0�Dw6�'y�(w6�=y�> y*�(y<�2Az�Bz �9B$z7�z2�:z2�z2�>z7�136u.�(u6�6137v�138v?�"v:�4w�:v?�?w�139w	�
w�140w(�+y*�0w6�6141y�A142y�143y�y�y�144y'	�"y*�*145y9�<146z�147z� 148z/�2z7�7149{�__main__rO)�__doc__�	streamlitr�r��pandasr��typingrrrr�io�	pdf_utilsrr�geometry_utilsr	r150r"�int�tupler,r�r3rJrfr�r�r�__name__rPr#r!�<module>rsj��151����,�,�	�B�)��$�t�*���h���,�D��T�#�u�*�5E��2��2��2�152�C��c��e��D)�d�8�n�)�d�3��8�n�)�_c�)�hl�)�b�G��H�~�G��G��G��	G�153�G�154�$�Z�
G�T/��H�~�/�155�t�*�/��/��	/�156157�$�Z�/�l]M�@	�z���F�r#