freeEDU/Log-Decoder

1"""2The interface for data preprocessing.3 4Authors:5    LogPAI Team6 7"""8 9 10import pandas as pd11import os12import numpy as np13import re14from collections import Counter15from scipy.special import expit16from itertools import compress17 18 19 20class FeatureExtractor(object):21 22    def __init__(self):23        self.idf_vec = None24        self.mean_vec = None25        self.events = None26        self.term_weighting = None27        self.normalization = None28        self.oov = None29 30    def fit_transform(self, X_seq, term_weighting=None, normalization=None, oov=False, min_count=1):31        """ Fit and transform the data matrix32 33        Arguments34        ---------35            X_seq: ndarray, log sequences matrix36            term_weighting: None or `tf-idf`37            normalization: None or `zero-mean`38            oov: bool, whether to use OOV event39            min_count: int, the minimal occurrence of events (default 0), only valid when oov=True.40 41        Returns42        -------43            X_new: The transformed data matrix44        """45        print('====== Transformed train data summary ======')46        self.term_weighting = term_weighting47        self.normalization = normalization48        self.oov = oov49 50        X_counts = []51        for i in range(X_seq.shape[0]):52            event_counts = Counter(X_seq[i])53            X_counts.append(event_counts)54        X_df = pd.DataFrame(X_counts)55        X_df = X_df.fillna(0)56        self.events = X_df.columns57        X = X_df.values58        if self.oov:59            oov_vec = np.zeros(X.shape[0])60            if min_count > 1:61                idx = np.sum(X > 0, axis=0) >= min_count62                oov_vec = np.sum(X[:, ~idx] > 0, axis=1)63                X = X[:, idx]64                self.events = np.array(X_df.columns)[idx].tolist()65            X = np.hstack([X, oov_vec.reshape(X.shape[0], 1)])66        67        num_instance, num_event = X.shape68        if self.term_weighting == 'tf-idf':69            df_vec = np.sum(X > 0, axis=0)70            self.idf_vec = np.log(num_instance / (df_vec + 1e-8))71            idf_matrix = X * np.tile(self.idf_vec, (num_instance, 1)) 72            X = idf_matrix73        if self.normalization == 'zero-mean':74            mean_vec = X.mean(axis=0)75            self.mean_vec = mean_vec.reshape(1, num_event)76            X = X - np.tile(self.mean_vec, (num_instance, 1))77        elif self.normalization == 'sigmoid':78            X[X != 0] = expit(X[X != 0])79        X_new = X80        81        print('Train data shape: {}-by-{}\n'.format(X_new.shape[0], X_new.shape[1])) 82        return X_new83 84    def transform(self, X_seq):85        """ Transform the data matrix with trained parameters86 87        Arguments88        ---------89            X: log sequences matrix90            term_weighting: None or `tf-idf`91 92        Returns93        -------94            X_new: The transformed data matrix95        """96        print('====== Transformed test data summary ======')97        X_counts = []98        for i in range(X_seq.shape[0]):99            event_counts = Counter(X_seq[i])100            X_counts.append(event_counts)101        X_df = pd.DataFrame(X_counts)102        X_df = X_df.fillna(0)103        empty_events = set(self.events) - set(X_df.columns)104        for event in empty_events:105            X_df[event] = [0] * len(X_df)106        X = X_df[self.events].values107        if self.oov:108            oov_vec = np.sum(X_df[X_df.columns.difference(self.events)].values > 0, axis=1)109            X = np.hstack([X, oov_vec.reshape(X.shape[0], 1)])110        111        num_instance, num_event = X.shape112        if self.term_weighting == 'tf-idf':113            idf_matrix = X * np.tile(self.idf_vec, (num_instance, 1)) 114            X = idf_matrix115        if self.normalization == 'zero-mean':116            X = X - np.tile(self.mean_vec, (num_instance, 1))117        elif self.normalization == 'sigmoid':118            X[X != 0] = expit(X[X != 0])119        X_new = X120 121        print('Test data shape: {}-by-{}\n'.format(X_new.shape[0], X_new.shape[1])) 122 123        return X_new, self.events124