kernelmachine/gpt3-quality-filter

1import gradio as gr2import os3import json4import numpy as np5from sklearn.feature_extraction.text import (CountVectorizer, TfidfTransformer, HashingVectorizer,6                                             TfidfVectorizer)7from sklearn.linear_model import LogisticRegression8from lr.hyperparameters import SEARCH_SPACE, RandomSearch, HyperparameterSearch9 10def load_model(serialization_dir):11    with open(os.path.join(serialization_dir, "best_hyperparameters.json"), 'r') as f:12        hyperparameters = json.load(f)13    if hyperparameters.pop('stopwords') == 1:14        stop_words = 'english'15    else:16        stop_words = None17    weight = hyperparameters.pop('weight')18    if weight == 'binary':19        binary = True20    else:21        binary = False22    ngram_range = hyperparameters.pop('ngram_range')23    ngram_range = sorted([int(x) for x in ngram_range.split()])24    if weight == 'tf-idf':25        vect = TfidfVectorizer(stop_words=stop_words,26                               lowercase=True,27                               ngram_range=ngram_range)28    elif weight == 'hash':29        vect = HashingVectorizer(stop_words=stop_words,lowercase=True,ngram_range=ngram_range)30    else:31        vect = CountVectorizer(binary=binary,32                               stop_words=stop_words,33                               lowercase=True,34                               ngram_range=ngram_range)35    if weight != "hash":36        with open(os.path.join(serialization_dir, "vocab.json"), 'r') as f:37            vocab = json.load(f)38        vect.vocabulary_ = vocab39    hyperparameters['C'] = float(hyperparameters['C'])40    hyperparameters['tol'] = float(hyperparameters['tol'])41    classifier = LogisticRegression(**hyperparameters)42    if os.path.exists(os.path.join(serialization_dir, "archive", "idf.npy")):43        vect.idf_ = np.load(os.path.join(serialization_dir,  "archive", "idf.npy"))44    classifier.coef_ = np.load(os.path.join(serialization_dir,  "archive", "coef.npy"))45    classifier.intercept_ = np.load(os.path.join(serialization_dir,  "archive", "intercept.npy"))46    classifier.classes_ = np.load(os.path.join(serialization_dir,  "archive", "classes.npy"))47    return classifier, vect48 49def score(x, clf, vectorizer):50    # score a single document51    return clf.predict_proba(vectorizer.transform([x]))52 53clf, vectorizer = load_model("model/")54 55def start(text):56    # print('yaya')57    # return 'hello'58    k = round(score(text, clf, vectorizer)[0][1], 2)59    return {"P(high quality)": k }60 61 62iface = gr.Interface(fn=start, inputs="text", outputs="key_values")63iface.launch()64