swethak21/tokenization

1import gradio as gr2from transformers import AutoTokenizer, AutoModel3import torch4 5tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")6model = AutoModel.from_pretrained("bert-base-uncased")7 8def process(text):9    inputs = tokenizer(text, return_tensors="pt")10 11    with torch.no_grad():12        outputs = model(**inputs)13 14    tokens = tokenizer.tokenize(text)15    shape = list(outputs.last_hidden_state.shape)16    embedding = outputs.last_hidden_state[0][0][:10].tolist()17 18    return tokens, shape, embedding19 20demo = gr.Interface(21    fn=process,22    inputs=gr.Textbox(label="Enter Text"),23    outputs=[24        gr.JSON(label="Tokens"),25        gr.JSON(label="Embedding Shape"),26        gr.JSON(label="First Token Embedding (First 10 Values)")27    ],28    title="Transformer Demo"29)30 31demo.launch()