mikachou/stackoverflow

1import numpy as np2from sklearn.feature_extraction.text import CountVectorizer3from sklearn.base import BaseEstimator, TransformerMixin4from sklearn.preprocessing import MultiLabelBinarizer5 6def top_topics(tags_list: iter, part: float) -> dict:7    cv = CountVectorizer(token_pattern='\S+')8    tags_vect = cv.fit_transform(tags_list)9    tags_vect_sum = np.sum(tags_vect.todense(), axis=0)10    return { k: v for (k, v) in sorted(list(zip(cv.get_feature_names_out(),np.array(tags_vect_sum)[0].tolist())), key=lambda tup: tup[1], reverse=True) if v >= part * len(list(tags_list)) }11 12def simplified_tags(orig_tags: list, allowed_tags: list, alternative: str = None, only_empty: bool = False) -> list:13    # intersection14    simplified_tags = list(set(orig_tags) & set(allowed_tags))15 16    # other missing tags = alternative param17    if alternative is not None:18        if (only_empty and len(simplified_tags) == 0) \19        or (not only_empty and len(simplified_tags) < len(orig_tags)):20            simplified_tags.append(alternative) # default = "other"21 22    return simplified_tags23 24class TagsSimplifier(BaseEstimator, TransformerMixin):25    def __init__(self, part=0.01):26        self.part = part27 28    def fit(self, X, y=None):29        self.count = top_topics(X, self.part)30        return self31 32    def transform(self, X, y=None):33        return X.apply(lambda tags: simplified_tags(tags.split(), self.count.keys())).values34 35    def inverse_transform(self, X, y=None):36        return X37 38class TagsBinarizer(BaseEstimator, TransformerMixin):39    def __init__(self, part=0.01):40        self.part = part41        self.ts = TagsSimplifier(part=self.part)42        self.mlb = MultiLabelBinarizer()43 44    def fit(self, X, y=None):45        simp_X = self.ts.fit_transform(X)46        self.mlb.fit(simp_X)47        return self48 49    def transform(self, X, y=None):50        simp_X = self.ts.transform(X)51        return self.mlb.transform(simp_X)52 53    def inverse_transform(self, X, y=None):54        return self.mlb.inverse_transform(X)