# Path: automlk/preprocessing.py
# class Transformer(object):
# class TransformerMissingFixed(Transformer):
# class TransformerNumericLog(Transformer):
# class TransformerNumericSqrt(Transformer):
# class TransformerDatetime(Transformer):
# class TransformerDate(Transformer):
# class TransformerDateWoYear(Transformer):
# class TransformerMissingFrequency(Transformer):
# class TransformerCategorical(Transformer):
# class TransformerLabel(TransformerCategorical):
# class TransformerOneHot(TransformerCategorical):
# class TransformerBaseN(TransformerCategorical):
# class TransformerHashing(TransformerCategorical):
# class TransformerBOW(Transformer):
# class TransformerWord2Vec(Transformer):
# class TransformerFastText(Transformer):
# class TransformerDoc2Vec(Transformer):
# class TransformerScaling(Transformer):
# class TransformerScalingStandard(TransformerScaling):
# class TransformerScalingMinMax(TransformerScaling):
# class TransformerScalingMaxAbs(TransformerScaling):
# class TransformerScalingRobust(TransformerScaling):
# class TransformerTruncatedSVD(Transformer):
# class TransformerFastICA(Transformer):
# class TransformerPCA(Transformer):
# class TransformerSelectFromModel(Transformer):
# class TransformerSelectionLinearSVR(TransformerSelectFromModel):
# class TransformerSelectionRfR(TransformerSelectFromModel):
# class TransformerSelectionRfC(TransformerSelectFromModel):
# class TransformerPassThrough(Transformer):
# class NoSampling(object):
#     def __init__(self, **params):
#     def set_params(self, **params):
#     def get_params(self, deep=True):
#     def get_feature_names(self):
#     def fit(self, X, y):
#     def transform(self, X):
#     def fit_transform(self, X, y):
#     def get_details(self):
#     def __init__(self, **params):
#     def transform(self, X):
#     def __init__(self, **params):
#     def transform(self, X):
#     def __init__(self, **params):
#     def transform(self, X):
#     def __init__(self, **params):
#     def transform(self, X):
#     def __init__(self, **params):
#     def transform(self, X):
#     def __init__(self, **params):
#     def transform(self, X):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def transform(self, X):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def transform(self, X):
#     def __init__(self, **params):
#     def __init__(self, **params):
#     def __init__(self, **params):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def transform(self, X):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def transform(self, X):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def transform(self, X):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def transform(self, X):
# def get_text_encoder(features, col, model_type, params):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def __init__(self, **params):
#     def __init__(self, **params):
#     def __init__(self, **params):
#     def __init__(self, **params):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def __init__(self, **params):
#     def __init__(self, **params):
#     def __init__(self, **params):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def transform(self, X):
#     def __init__(self, **params):
#     def fit_sample(self, X, y):
#             T = pd.DataFrame(encoder.transform(text).todense()).reset_index(drop=True)
#             X = pd.concat([X.reset_index(drop=True), T], axis=1)
#             T = pd.DataFrame(vector_word2vec(encoder, X[col].values, self.params)).reset_index(drop=True)
#             X = pd.concat([X.reset_index(drop=True), T], axis=1)
#             T = pd.DataFrame(vector_fasttext(encoder, X[col].values, self.params)).reset_index(drop=True)
#             X = pd.concat([X.reset_index(drop=True), T], axis=1)
#             T = pd.DataFrame(vector_doc2vec(encoder, X[col].values, self.params)).reset_index(drop=True)
#             X = pd.concat([X.reset_index(drop=True), T], axis=1)


import string
import pandas as pd
import numpy as np
from random import shuffle
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import log_loss
from gensim.models import Word2Vec, Doc2Vec
from gensim.models.wrappers.fasttext import FastText
from gensim.models.doc2vec import TaggedDocument
from sklearn.linear_model import LogisticRegression
from automlk.preprocessing import clean_text


df = pd.read_csv('../datasets/imdb/labeledTrainData.tsv', sep='\t', header=0)

text = df['review'].values
y = df['sentiment'].values

mode = 'w2v'

if mode == 'bow':
    # --------------------------------------------------------------------
    # Bag of words

# Path: automlk/preprocessing.py
# class Transformer(object):
# class TransformerMissingFixed(Transformer):
# class TransformerNumericLog(Transformer):
# class TransformerNumericSqrt(Transformer):
# class TransformerDatetime(Transformer):
# class TransformerDate(Transformer):
# class TransformerDateWoYear(Transformer):
# class TransformerMissingFrequency(Transformer):
# class TransformerCategorical(Transformer):
# class TransformerLabel(TransformerCategorical):
# class TransformerOneHot(TransformerCategorical):
# class TransformerBaseN(TransformerCategorical):
# class TransformerHashing(TransformerCategorical):
# class TransformerBOW(Transformer):
# class TransformerWord2Vec(Transformer):
# class TransformerFastText(Transformer):
# class TransformerDoc2Vec(Transformer):
# class TransformerScaling(Transformer):
# class TransformerScalingStandard(TransformerScaling):
# class TransformerScalingMinMax(TransformerScaling):
# class TransformerScalingMaxAbs(TransformerScaling):
# class TransformerScalingRobust(TransformerScaling):
# class TransformerTruncatedSVD(Transformer):
# class TransformerFastICA(Transformer):
# class TransformerPCA(Transformer):
# class TransformerSelectFromModel(Transformer):
# class TransformerSelectionLinearSVR(TransformerSelectFromModel):
# class TransformerSelectionRfR(TransformerSelectFromModel):
# class TransformerSelectionRfC(TransformerSelectFromModel):
# class TransformerPassThrough(Transformer):
# class NoSampling(object):
#     def __init__(self, **params):
#     def set_params(self, **params):
#     def get_params(self, deep=True):
#     def get_feature_names(self):
#     def fit(self, X, y):
#     def transform(self, X):
#     def fit_transform(self, X, y):
#     def get_details(self):
#     def __init__(self, **params):
#     def transform(self, X):
#     def __init__(self, **params):
#     def transform(self, X):
#     def __init__(self, **params):
#     def transform(self, X):
#     def __init__(self, **params):
#     def transform(self, X):
#     def __init__(self, **params):
#     def transform(self, X):
#     def __init__(self, **params):
#     def transform(self, X):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def transform(self, X):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def transform(self, X):
#     def __init__(self, **params):
#     def __init__(self, **params):
#     def __init__(self, **params):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def transform(self, X):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def transform(self, X):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def transform(self, X):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def transform(self, X):
# def get_text_encoder(features, col, model_type, params):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def __init__(self, **params):
#     def __init__(self, **params):
#     def __init__(self, **params):
#     def __init__(self, **params):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def __init__(self, **params):
#     def __init__(self, **params):
#     def __init__(self, **params):
#     def __init__(self, **params):
#     def fit(self, X, y):
#     def transform(self, X):
#     def __init__(self, **params):
#     def fit_sample(self, X, y):
#             T = pd.DataFrame(encoder.transform(text).todense()).reset_index(drop=True)
#             X = pd.concat([X.reset_index(drop=True), T], axis=1)
#             T = pd.DataFrame(vector_word2vec(encoder, X[col].values, self.params)).reset_index(drop=True)
#             X = pd.concat([X.reset_index(drop=True), T], axis=1)
#             T = pd.DataFrame(vector_fasttext(encoder, X[col].values, self.params)).reset_index(drop=True)
#             X = pd.concat([X.reset_index(drop=True), T], axis=1)
#             T = pd.DataFrame(vector_doc2vec(encoder, X[col].values, self.params)).reset_index(drop=True)
#             X = pd.concat([X.reset_index(drop=True), T], axis=1)


# Path: tests/test_imdb.py
import string
import pandas as pd
import numpy as np
from random import shuffle
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import log_loss
from gensim.models import Word2Vec, Doc2Vec
from gensim.models.wrappers.fasttext import FastText
from gensim.models.doc2vec import TaggedDocument
from sklearn.linear_model import LogisticRegression
from automlk.preprocessing import clean_text



df = pd.read_csv('../datasets/imdb/labeledTrainData.tsv', sep='\t', header=0)

text = df['review'].values
y = df['sentiment'].values

mode = 'w2v'

if mode == 'bow':
    # --------------------------------------------------------------------
    # Bag of words

    text = [clean_text(s) for s in text]