huggingface/Model_Cards_Writing_Tool
125
1import re2import streamlit as st3from modelcards import CardData, ModelCard4from markdownTagExtract import tag_checker,listToString,to_markdown5#from specific_extraction import extract_it6 7 8# from persist import persist9#global bytes_data10 11 12################################################################13#### Markdown parser logic #################################14################################################################15 16def file_upload():17 bytes_data = st.session_state.markdown_upload18 return bytes_data19 20 21# Sets up the basics22model_card_md = file_upload() # this is where the new model card will be read in from23model_card_md = model_card_md#.decode("utf-8")24# Does metadata appear in any other format than this?25metadata_re = re.compile("^---(.*?)---", re.DOTALL)26header_re = re.compile("^\s*# (.*)", re.MULTILINE)27subheader_re = re.compile("^\s*## (.*)", re.MULTILINE)28subsubheader_re = re.compile("^\s*### (.*)", re.MULTILINE)29subsubsubheader_re = re.compile("^\s*#### (.*)", re.MULTILINE)30# We could be a lot more flexible on this re.31# We require keys to be bold-faced here.32# We don't have to require bold, as long as it's key:value33# **License:**34# Bold terms use ** or __35# Allows the mixing of ** and __ for bold but eh whatev36key_value_re = re.compile("^\s*([*_]{2}[^*_]+[*_]{2})([^\n]*)", re.MULTILINE)37# Hyphens or stars mark list items.38# Unordered list39list_item_re = re.compile("^\s*[-*+]\s+.*", re.MULTILINE)40# This is the ordered list41enum_re = re.compile("^\s*[0-9].*", re.MULTILINE)42table_re = re.compile("^\s*\|.*", re.MULTILINE)43text_item_re = re.compile("^\s*[A-Za-z(](.*)", re.MULTILINE)44# text_item_re = re.compile("^\s*#\s*.*", re.MULTILINE)45# Allows the mixing of -* and *- for italics but eh whatev46italicized_text_item_re = re.compile(47 "^[_*][^_*\s].*\n?.*[^_*][_*]$", flags=re.MULTILINE48)49tag_re = re.compile("^\s*<.*", re.MULTILINE)50image_re = re.compile("!\[.*\]\(.*\)", re.MULTILINE)51 52 53subheader_re_dict = {}54subheader_re_dict[header_re] = subheader_re55subheader_re_dict[subheader_re] = subsubheader_re56subheader_re_dict[subsubheader_re] = subsubsubheader_re57 58 59def get_metadata(section_text):60 return list(metadata_re.finditer(section_text))61 62 63def find_images(section_text):64 return list(image_re.finditer(section_text))65 66 67def find_tags(section_text):68 return list(tag_re.finditer(section_text))69 70 71def find_tables(section_text):72 return list(table_re.finditer(section_text))73 74 75def find_enums(section_text):76 return list(enum_re.finditer(section_text))77 78 79# Extracts the stuff from the .md file80def find_key_values(section_text):81 return list(key_value_re.finditer(section_text))82 83 84def find_lists(section_text):85 # Find lists: Those lines starting with either '-' or '*'86 return list(list_item_re.finditer(section_text))87 88 89def find_texts(section_text):90 # Find texts: Free writing within a section91 basic_text = list(text_item_re.finditer(section_text))92 ital_text = list(italicized_text_item_re.finditer(section_text))93 free_text = basic_text + ital_text94 return free_text95 96 97def find_headers(full_text):98 headers = list(header_re.finditer(full_text))99 subheaders = list(subheader_re.finditer(full_text))100 subsubheaders = list(subsubheader_re.finditer(full_text))101 subsubsubheaders = list(subsubsubheader_re.finditer(full_text))102 return (headers, subheaders, subsubheaders, subsubsubheaders)103 104 105metadata_list = get_metadata(model_card_md)106if metadata_list != []:107 metadata_end = metadata_list[-1].span()[-1]108 print("Metadata extracted")109 # Metadata processing can happen here.110 # For now I'm just ignoring it.111 model_card_md = model_card_md[metadata_end:]112else:113 print("No metadata found")114 115# Matches of all header types116headers_list = find_headers(model_card_md)117print("Headers extracted")118# This type of header (one #)119headers = headers_list[0]120## This type of header (two ##)121subheaders = headers_list[1]122### This type of header123subsubheaders = headers_list[2]124#### This type of header125subsubsubheaders = headers_list[3]126 127# Matches of bulleted lists128lists_list = find_lists(model_card_md)129print("Bulleted lists extracted")130 131enums_list = find_enums(model_card_md)132print("Enumerated lists extracted")133 134key_value_list = find_key_values(model_card_md)135print("Key values extracted")136 137tables_list = find_tables(model_card_md)138print("Tables extracted")139 140tags_list = find_tags(model_card_md)141print("Markup tags extracted")142 143images_list = find_images(model_card_md)144print("Images extracted")145 146# Matches of free text within a section147texts_list = find_texts(model_card_md)148print("Free text extracted")149 150 151# List items have the attribute: value;152# This provides for special handling of those strings,153# allowing us to check if it's a list item in order to split/print ok.154LIST_ITEM = "List item"155KEY_VALUE = "Key: Value"156FREE_TEXT = "Free text"157ENUM_LIST_ITEM = "Enum item"158TABLE_ITEM = "Table item"159TAG_ITEM = "Markup tag"160IMAGE_ITEM = "Image"161 162 163def create_span_dict(match_list, match_type):164 """165 Creates a dictionary made out of all the spans.166 This is useful for knowing which types to fill out with what in the app.167 Also useful for checking if there are spans in the .md file that we've missed.168 """169 span_dict = {}170 for match in match_list:171 if len(match.group().strip()) > 0:172 span_dict[(match.span())] = (match.group(), match_type)173 return span_dict174 175 176metadata_span_dict = create_span_dict(metadata_list, "Metadata")177# Makes a little dict for each span type178header_span_dict = create_span_dict(headers, "# Header")179subheader_span_dict = create_span_dict(subheaders, "## Subheader")180subsubheader_span_dict = create_span_dict(subsubheaders, "### Subsubheader")181subsubsubheader_span_dict = create_span_dict(subsubsubheaders, "#### Subsubsubheader")182key_value_span_dict = create_span_dict(key_value_list, KEY_VALUE)183lists_span_dict = create_span_dict(lists_list, LIST_ITEM)184enums_span_dict = create_span_dict(enums_list, ENUM_LIST_ITEM)185tables_span_dict = create_span_dict(tables_list, TABLE_ITEM)186tags_span_dict = create_span_dict(tags_list, TAG_ITEM)187images_span_dict = create_span_dict(images_list, IMAGE_ITEM)188texts_span_dict = create_span_dict(texts_list, FREE_TEXT)189 190# We don't have to have these organized by type necessarily.191# Doing it here for clarity.192all_spans_dict = {}193all_spans_dict["headers"] = header_span_dict194all_spans_dict["subheaders"] = subheader_span_dict195all_spans_dict["subsubheaders"] = subsubheader_span_dict196all_spans_dict["subsubsubheaders"] = subsubsubheader_span_dict197all_spans_dict[LIST_ITEM] = lists_span_dict198all_spans_dict[KEY_VALUE] = key_value_span_dict199all_spans_dict[TABLE_ITEM] = tables_span_dict200all_spans_dict[ENUM_LIST_ITEM] = enums_span_dict201all_spans_dict[TAG_ITEM] = tags_span_dict202all_spans_dict[IMAGE_ITEM] = images_span_dict203all_spans_dict[FREE_TEXT] = texts_span_dict204 205 206def get_sorted_spans(spans_dict):207 merged_spans = {}208 for span_dict in spans_dict.values():209 merged_spans.update(span_dict)210 sorted_spans = sorted(merged_spans)211 return sorted_spans, merged_spans212 213 214sorted_spans, merged_spans = get_sorted_spans(all_spans_dict)215 216# Sanity/Parse check. Have we captured all spans in the .md file?217if sorted_spans[0][0] != 0:218 print("FYI, our spans don't start at the start of the file.")219 print("We did not catch this start:")220 print(model_card_md[: sorted_spans[0][0]])221 222for idx in range(len(sorted_spans) - 1):223 last_span_end = sorted_spans[idx][1]224 new_span_start = sorted_spans[idx + 1][0]225 if new_span_start > last_span_end + 1:226 start_nonparse = sorted_spans[idx]227 end_nonparse = sorted_spans[idx + 1]228 text = model_card_md[start_nonparse[1] : end_nonparse[0]]229 if text.strip():230 print("Found an unparsed span in the file:")231 print(start_nonparse)232 print(" ---> ")233 print(end_nonparse)234 print(text)235 236# print(header_span_dict)237def section_map_to_help_text(text_retrieved):238 239 presit_states = {240 "## Model Details": "Give an overview of your model, the relevant research paper, who trained it, etc.",241 "## How to Get Started with the Model": "Give an overview of how to get started with the model",242 "## Limitations and Biases": "Provide an overview of the possible Limitations and Risks that may be associated with this model",243 "## Uses": "Detail the potential uses, intended use and out-of-scope uses for this model",244 "## Training": "Provide an overview of the Training Data and Training Procedure for this model",245 "## Evaluation Results": "Detail the Evaluation Results for this model",246 "## Environmental Impact": "Provide an estimate for the carbon emissions: Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here.",247 "## Citation Information": "How to best cite the model authors",248 "## Glossary": "If relevant, include terms and calculations in this section that can help readers understand the model or model card.",249 "## More Information": "Any additional information",250 "## Model Card Authors": "This section provides another layer of transparency and accountability. Whose views is this model card representing? How many voices were included in its construction? Etc.",251 "Model Card Contact": "Mediums to use, in order to contact the model creators",252 "## Technical Specifications": " Additional technical information",253 '## Model Examination': " Examining the model",254 }255 256 for key in presit_states:257 if key == text_retrieved:258 return presit_states(key)259 260 261def section_map_to_persist(text_retrieved):262 263 presit_states = {264 "Model_details_text": "## Model Details",265 "Model_how_to": "## How to Get Started with the Model",266 "Model_Limits_n_Risks": "## Limitations and Biases",267 "Model_uses": "## Uses",268 "Model_training": "## Training",269 "Model_Eval": "## Evaluation Results",270 "Model_carbon": "## Environmental Impact",271 "Model_cite": "## Citation Information",272 "Glossary": "## Glossary",273 "More_info": "## More Information",274 "Model_card_authors": "## Model Card Authors",275 "Model_card_contact": "## Model Card Contact",276 "Technical_specs": "## Technical specifications",277 "Model_examin": "## Model Examination",278 }279 280 for key in presit_states:281 if presit_states[key] == text_retrieved:282 return key283 284 285def main():286 # st.write('here')287 print(extract_it("Model_details_text"))288 289 290def extract_headers():291 headers = {}292 subheaders = {}293 subsubheaders = {}294 subsubsubheaders = {}295 previous = (None, None, None, None)296 297 for s in sorted_spans:298 if merged_spans[s][1] == "# Header":299 headers[s] = (sorted_spans.index(s), previous[0])300 previous = (sorted_spans.index(s), previous[1], previous[2], previous[3])301 if merged_spans[s][1] == "## Subheader":302 subheaders[s] = (sorted_spans.index(s), previous[1])303 previous = (previous[0], sorted_spans.index(s), previous[2], previous[3])304 if merged_spans[s][1] == "### Subsubheader":305 subsubheaders[s] = (sorted_spans.index(s), previous[2])306 previous = (previous[0], previous[1], sorted_spans.index(s), previous[3])307 if merged_spans[s][1] == "#### Subsubsubheader":308 subsubsubheaders[s] = (sorted_spans.index(s), previous[3])309 previous = (previous[0], previous[1], previous[2], sorted_spans.index(s))310 311 return headers, subheaders, subsubheaders, subsubsubheaders312 313 314def stringify():315 headers, subheaders, subsubheaders, subsubsubheaders = extract_headers()316 headers_strings = {}317 subheaders_strings = {}318 subsubheaders_strings = {}319 subsubsubheaders_strings = {}320 321 first = None322 for i in headers:323 if headers[i][1] == None:324 continue325 sub_spans = sorted_spans[headers[i][1] : headers[i][0]]326 lines = []327 for x in sub_spans:328 lines.append(merged_spans[x][0])329 try:330 name = lines[0]331 except:332 name = "Model Details"333 lines = "".join(lines)334 # print(merged_spans[i][0] + "-------------------")335 # print(lines)336 headers_strings[337 name.replace("\n# ", "")338 .replace(" ", "")339 .replace(" ", "")340 .replace("\n", "")341 .replace("{{", "")342 .replace("}}", "")343 ] = lines344 first = i345 346 first = None347 for i in subheaders:348 if subheaders[i][1] == None:349 continue350 sub_spans = sorted_spans[subheaders[i][1] : subheaders[i][0]]351 lines = []352 for x in sub_spans:353 if merged_spans[x][1] == "## Subheader" and first == None:354 break355 elif merged_spans[x][1] == "# Header":356 break357 else:358 lines.append(merged_spans[x][0])359 try:360 name = lines[0]361 except:362 name = "Model Details"363 lines = "".join(lines)364 # print(merged_spans[i][0] + "-------------------")365 # print(lines)366 subheaders_strings[367 name.replace("\n# ", "").replace(" ", "").replace(" ", "")368 ] = lines369 first = i370 371 first = None372 for i in subsubheaders:373 if subsubheaders[i][1] == None:374 continue375 sub_spans = sorted_spans[subsubheaders[i][1] : subsubheaders[i][0]]376 lines = []377 for x in sub_spans:378 if merged_spans[x][1] == "## Subheader" or (379 merged_spans[x][1] == "### Subsubheader" and first == None380 ):381 break382 else:383 lines.append(merged_spans[x][0])384 lines = "".join(lines)385 386 subsubheaders_strings[387 merged_spans[i][0].replace("\n", "").replace("### ", "").replace(" ", "")388 ] = lines389 first = i390 391 for i in subsubsubheaders:392 if subsubsubheaders[i][1] == None:393 continue394 sub_spans = sorted_spans[subsubsubheaders[i][1] : subsubsubheaders[i][0]]395 lines = []396 for x in sub_spans:397 if (398 merged_spans[x][1] == "## Subheader"399 or merged_spans[x][1] == "### Subsubheader"400 ):401 break402 else:403 lines.append(merged_spans[x][0])404 lines = "".join(lines)405 406 subsubsubheaders_strings[407 merged_spans[i][0].replace("#### ", "").replace("**", "").replace("\n", "")408 ] = lines409 410 return (411 headers_strings,412 subheaders_strings,413 subsubheaders_strings,414 subsubsubheaders_strings,415 )416 417 418def extract_it(text_to_retrieve):419 print("Span\t\tType\t\tText")420 print("-------------------------------------")421 found_subheader = False422 current_subheader = " "423 page_state = " "424 help_text = " "425 #st.write("in cs- body here")426 427 (428 headers_strings,429 subheaders_strings,430 subsubheaders_strings,431 subsubsubheaders_strings,432 ) = stringify()433 434 h_keys = list(headers_strings.keys())435 sh_keys = list(subheaders_strings.keys())436 ssh_keys = list(subsubheaders_strings.keys())437 sssh_keys = list(subsubsubheaders_strings.keys())438 439 needed = [440 "model details",441 "howto",442 "limitations",443 "uses",444 "training",445 "evaluation",446 "environmental",447 "citation",448 "glossary",449 "more information",450 "authors",451 "contact",452 ] # not sure what keyword should be used for citation, howto, and contact453 # info_strings = {454 # "details": "## Model Details",455 # "howto": "## How to Get Started with the Model",456 # "limitations": "## Limitations and Biases",457 # "uses": "## Uses",458 # "training": "## Training",459 # "evaluation": "## Evaluation Results",460 # "environmental": "## Environmental Impact",461 # "citation": "## Citation Information",462 # "glossary": "## Glossary",463 # "more information": "## More Information",464 # "authors": "## Model Card Authors",465 # "contact": "## Model Card Contact",466 # }467 info_strings = {468 "model details": "",469 "howto": "",470 "limitations": "",471 "uses": "",472 "training": "",473 "evaluation": "",474 "environmental": "",475 "citation": "",476 "glossary": "",477 "more information": "",478 "authors": "",479 "contact": "",480 }481 482 for x in needed:483 for l in h_keys:484 if x in l.lower():485 info_strings[x] = info_strings[x] + headers_strings[l]486 for i in sh_keys:487 if x in i.lower():488 info_strings[x] = info_strings[x] + subheaders_strings[i]489 for z in ssh_keys:490 try:491 if x in z.lower():492 info_strings[x] = info_strings[x] + subsubheaders_strings[z]493 except:494 continue495 for y in sssh_keys:496 try:497 if x in y.lower():498 info_strings[x] = info_strings[x] + subsubsubheaders_strings[y]499 except:500 continue501 502 extracted_info = {503 "Model_details_text": info_strings["model details"],504 "Model_how_to": info_strings["howto"],505 "Model_Limits_n_Risks": info_strings["limitations"],506 "Model_uses": info_strings["uses"],507 "Model_training": info_strings["training"],508 "Model_Eval": info_strings["evaluation"],509 "Model_carbon": info_strings["environmental"],510 "Model_cite": info_strings["citation"],511 "Glossary": info_strings["glossary"],512 "More_info": info_strings["more information"],513 "Model_card_authors": info_strings["authors"],514 "Model_card_contact": info_strings["contact"],515 "Technical_specs": "## Technical specifications",516 "Model_examin": "## Model Examination",517 }518 519 #text_to_retrieve = "Model_details_text"520 521 new_t = extracted_info[text_to_retrieve] + " "522 523 return(new_t)524 525 526if __name__ == "__main__":527 528 main()529