<jupyter_start><jupyter_text>Language Model with label smoothing<jupyter_code>path = Path('/home/jupyter/insight_project/Project-M/data/preprocessed/csv')
filenames = path.ls()
len(filenames)
dfs = []
i=0
for file in filenames:
    if file.name == 'models': continue
    df = pd.read_csv(file)
    df['user'] = i
    i = i+1 
    dfs.append(df)

df_total = pd.concat(dfs)
df_total.tail()
dfs[0]['Label'].value_counts().plot(kind='bar')
ax = df_total.groupby(['Label','user']).size().unstack().plot(kind='bar',stacked=True,
                                                         legend=False,title='Total Data Distribution'
                                                        )
ax.set(xlabel='Relevent', ylabel='Counts')
plt.show()
df_total['Label'].value_counts()
df_total.iloc[400]['Body']
txt_proc = [
    OpenFileProcessor(),
    TokenizeProcessor(),
    NumericalizeProcessor(min_freq=2)
]
data_lm = (TextList
           .from_df(df_total,path=path,processor=txt_proc)
           .split_by_rand_pct(0.1)
           .label_for_lm()
           .databunch(bs=64,num_workers=os.cpu_count()*4)
          )<jupyter_output><empty_output><jupyter_text>Pre-processing:* pre_rules:* post_rules:<jupyter_code>pd.set_option('display.max_colwidth', -1)<jupyter_output><empty_output><jupyter_text>Original text and after pre-processing / tokenized text<jupyter_code>data_lm.train_ds.x.inner_df.iloc[140]['Body']
data_lm.train_ds[140]
data_lm.train_ds.x.inner_df.iloc[570]['Body']
data_lm.train_ds[570]
data_lm.vocab.stoi['in.fitsapp.sheliza.fitsapp']
data_lm.save(path/'models/data_lm_large.pkl')
data_lm = load_data(path/'models','data_lm_large.pkl',bs=64)
loss_func = FlattenedLoss(LabelSmoothingCrossEntropy)
learn = language_model_learner(data_lm,AWD_LSTM,loss_func=loss_func,drop_mult=0.3)
learn.to_fp16()
learn.lr_find()
learn.recorder.plot(suggestion=True)
learn.fit_one_cycle(1,1e-2,moms=(0.8,0.7))
learn.unfreeze()
learn.lr_find()
learn.recorder.plot(suggestion=True)
learn.fit_one_cycle(10,1e-3,moms=(0.8,0.7))
learn.save('fine-tuned-large-fwd')
learn.to_fp32().save(path/'models/fine-tuned-fwd',with_opt=False)
learn.save_encoder(path/'models/fine-tuned-enc-fwd')
!ls {path}/'models'<jupyter_output>1a.pth			    data_lm.pkl		    models
awd_lstm_baseline-0917.pth  fine-tuned-enc-fwd.pth  textlist_class_seed42
data_lm_large.pkl	    fine-tuned-fwd.pth	    tmp.pth<jupyter_text>Backward language model<jupyter_code>txt_proc = [
    OpenFileProcessor(),
    TokenizeProcessor(),
    NumericalizeProcessor(min_freq=2)
]
data_lm = (TextList
           .from_df(df_total,path=path,processor=txt_proc)
           .split_by_rand_pct(0.1)
           .label_for_lm()
           .databunch(bs=64,num_workers=os.cpu_count()*4,backwards=True)
          )
data_lm.save(path/'models/data_lm_large_bwd.pkl')
data_lm = load_data(path/'models/','data_lm_large_bwd.pkl',bs=64,backwards=True)
loss_func = FlattenedLoss(LabelSmoothingCrossEntropy)
learn = language_model_learner(data_lm,AWD_LSTM,loss_func=loss_func,drop_mult=0.5).to_fp16()
learn.lr_find()
learn.recorder.plot(suggestion=True)
lr = 1e-2
learn.fit_one_cycle(2, lr, moms=(0.8,0.7))
learn.unfreeze()
learn.lr_find()
learn.fit_one_cycle(10, lr/10, moms=(0.8,0.7))
learn.to_fp32().save(path/'models/fine-tuned-bwd',with_opt=False)
learn.save_encoder(path/'models/fine-tuned-enc-bwd')
!ls {path}/models<jupyter_output>1a.pth			    data_lm.pkl		    fine-tuned-fwd.pth
awd_lstm_baseline-0917.pth  fine-tuned-bwd.pth	    models
data_lm_large_bwd.pkl	    fine-tuned-enc-bwd.pth  textlist_class_seed42
data_lm_large.pkl	    fine-tuned-enc-fwd.pth  tmp.pth<jupyter_text>Classifier - FWD<jupyter_code>filenames.remove(Path('/home/jupyter/insight_project/Project-M/data/preprocessed/csv/models'))
len(filenames)
import random

random.seed(42)
random.shuffle(filenames)
 
def _train_valid_split(total_filenames,train_pct=0.7):
    '''split the train and test base on total datasets
        return train_list and valid_list with their associated path'''

    split_point = round(len(total_filenames)*train_pct)
    assert (type(split_point) == int)
    return total_filenames[:split_point],total_filenames[split_point:]

def _get_df_from_filenames(filenames):
    '''pass a list of filenames, return df of total'''
    
    dfs = []
    for file in filenames:
        df = pd.read_csv(file)
        dfs.append(df)

    df_total = pd.concat(dfs)
    return df_total
train_list, valid_list = _train_valid_split(filenames)
df_train = _get_df_from_filenames(train_list)
df_train['Label'].value_counts()
!ls {path}/'models'
data_lm = load_data(path/'models','data_lm_large.pkl',bs=64)
data_lm.vocab.stoi['this']
data = (TextList
        .from_df(df=df_train,path=path,cols='Body',vocab=data_lm.vocab)
        .split_by_rand_pct(0.1,seed=42)
        .label_from_df(cols='Label')
        .databunch(bs=64,num_workers=os.cpu_count()*4)
       )
data.vocab.stoi['bigquery']
data.train_ds.inner_df.iloc[17]['Body']
data.train_ds[14][0]
data.train_ds.inner_df.iloc[17]['Label']
from sklearn.metrics import f1_score

@np_func
def f1(inp,targ): return f1_score(targ, np.argmax(inp, axis=-1))
loss_func = FlattenedLoss(LabelSmoothingCrossEntropy)
learn = text_classifier_learner(data,AWD_LSTM,drop_mult=0.5,
                                loss_func=loss_func,
                                metrics=[accuracy,f1], #FBeta(beta=1)
                                model_dir = path/'models'
                               ).to_fp16()
learn.load_encoder('fine-tuned-enc-fwd');
learn.freeze_to(-1);
learn.lr_find()
learn.recorder.plot(suggestion=True)
lr = 4e-2 * 64/48
moms = (0.8,0.7)
learn.fit_one_cycle(2,lr,moms=moms)
learn.fit_one_cycle(2,lr,moms=moms)
learn.save('1a')
learn.freeze_to(-2)
learn.fit_one_cycle(2,slice(lr/(2.6**4),lr),moms=moms)
learn.freeze_to(-3)
learn.fit_one_cycle(2,slice(lr/2/(2.6**4),lr),moms=moms)
learn.unfreeze()
learn.fit_one_cycle(1,slice(lr/10/(2.6**4),lr/10),moms=moms)
learn.save('general-clasifier-0.84')
!ls {path}/models
def _get_individual_data(filepath):
    df = pd.read_csv(filepath)
    if df.shape[0]*0.7 < 64: 
        bs= int(df.shape[0]*0.7)
    else: bs=64
    data = (TextList
            .from_df(df=df,path=path,cols='Body',vocab=data_lm.vocab)
            .split_by_rand_pct(0.3,seed=42)
            .label_from_df(cols='Label')
            .databunch(bs=bs,num_workers=os.cpu_count()*4)
           )
    return data

def _get_individual_learner(data):
    learn = text_classifier_learner(data,AWD_LSTM,drop_mult=0.5,
                                loss_func=FlattenedLoss(LabelSmoothingCrossEntropy),
                                metrics=[accuracy], #FBeta(beta=1)
                                model_dir = path/'models'
                               )
    if torch.cuda.is_available():
        learn.to_fp16()
    learn.load('general-clasifier-0.84')
    learn.freeze_to(-1)
    return learn

def run_valid(data_path):
    _data_valid = _get_individual_data(data_path)
    _learn_valid = _get_individual_learner(_data_valid)
    _learn_valid.fit_one_cycle(1,1e-1,moms=(0.8,0.7))
    return _learn_valid.recorder.metrics[0][0].item()
%%time
valid_acc = []
for file in valid_list:
    valid_acc.append(run_valid(file))
valid_acc
sum(valid_acc) / 21<jupyter_output><empty_output><jupyter_text>Compare with Basilica baseline<jupyter_code>valid_list[0].name.split('.')[0]
valid_list[0]
test_path = Path('/home/jupyter/insight_project/Project-M/data/raw/' 
                 + valid_list[0].name.split('.')[0]
                 + '.json'
                )
test_path
def get_json_valid(valid_lists):
    json_path = []
    for path in valid_lists:
        json_path.append(Path('/home/jupyter/insight_project/Project-M/data/raw/'
                              + path.name.split('.')[0]
                              + '.json')
                        )
    return json_path
json_list = get_json_valid(valid_list)
import json
import sklearn.decomposition
import sklearn.linear_model
import sklearn.preprocessing
def data_point_features(data_point):
    return data_point['embedding'] + source_features(data_point['source'])

def source_features(source):
    return [int(source == 'hn'), int(source == 'reddit'), int(source == 'twitter'),
            int(source not in ['hn', 'reddit', 'twitter'])]
test_score_basilica = []
def quick_run(obj):   
    random.seed(120)
    random.shuffle(obj['data_points'])
    features = []
    labels = []
    for data_point in obj['data_points']:
        features.append(data_point_features(data_point))
        labels.append(data_point['label'])
    train_sz = int(len(features)*0.7)
    print('  Splitting into train (%s) and test (%s)....' % (
        train_sz, len(features) - train_sz))
    train_features = features[:train_sz]
    train_labels = labels[:train_sz]
    test_features = features[train_sz:]
    test_labels = labels[train_sz:]
    # if 0 in train_labels and 1 in train_labels:
    #     break
    print('  Preprocessing features...')
    fixed_train_features = (np.array(train_features))
    fixed_test_features = (np.array(test_features))
        # We scale the features to have mean 0 and unit standard
        # deviation.  We always want mean 0 to stop us from wasting the
        # first PCA component.  Scaling the standard deviations is more
        # questionable, it might be worth experimenting with removing
        # that.
    scaler = sklearn.preprocessing.StandardScaler(with_std=True)
    fixed_train_features = scaler.fit_transform(fixed_train_features)
    fixed_test_features = scaler.transform(fixed_test_features)
        # If we have a small number of data points, PCA down to half the
        # number of data points.  The exact PCA size here could probably
        # be tweaked, and I'm not sure whether or not whitening is
        # correct.
    pca_size = min(len(train_features)/2, len(train_features[0]))
    pca = sklearn.decomposition.PCA(n_components=int(pca_size), whiten=True)
    fixed_train_features = pca.fit_transform(fixed_train_features)
    fixed_test_features = pca.transform(fixed_test_features)

    print('  Training model...')
        # We train a bog-standard logistic regression, and it works fairly
        # well.
    classifier = sklearn.linear_model.LogisticRegression()
    classifier.fit(fixed_train_features, np.array(train_labels))
    train_score = classifier.score(fixed_train_features, np.array(train_labels))
    test_score = classifier.score(fixed_test_features, np.array(test_labels))

    print('  Train score: %s.' % train_score)
    print('  Test score: %s.' % test_score)
    test_score_basilica.append(test_score)
%%time
for json_path in json_list:
    with open(json_path) as f:
        obj = json.load(f)
        quick_run(obj)
#test_score_basilica
test_score_basilica
sum(test_score_basilica) / 21
sum(test_score_basilica) / 21<jupyter_output><empty_output><jupyter_text>Try KNN<jupyter_code>from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score

def get_emb(df):
    emb = []
    for i in df['Emb'].values:
        emb.append(np.array(i))
    return emb

def run_knn(one_file):
    body, label, source, emb = [], [], [], []
    with open(one_file) as file:
        data = json.load(file)
    for item in data['data_points']:
        body.append(item['body'])
        label.append(item['label'])
        source.append(item['source'])
        emb.append(item['embedding'])
    df = pd.DataFrame(list(zip(body, label, source, emb)),
                         columns = ['Body', 'Label', 'Source', 'Emb'])
    df = df.sample(frac=1, random_state=100).reset_index(drop=True)
    i = round(df.shape[0] * 0.7)
    df_train = df[:i]
    df_valid = df[i:]
    train_emb = get_emb(df_train)
    valid_emb = get_emb(df_valid)
    np_train_x = np.array(train_emb)
    np_train_y = np.array(df_train['Label'].values)
    np_valid_x = np.array(valid_emb)
    np_valid_y = np.array(df_valid['Label'].values)
    neigh.fit(np_train_x,np_train_y)
    preds = neigh.predict(np_valid_x)
    valid_score = accuracy_score(np_valid_y,preds)
    print(f'Test Score: {valid_score:.2f}')
    return valid_score
    
%%time
knn_search = []
knn_std = []
for i in range(1,41):
    neigh = KNeighborsClassifier(n_neighbors=i)
    knn_test_score_basilica = []
    for json_path in json_list:
        knn_test_score_basilica.append(run_knn(json_path))
    knn_search.append(sum(knn_test_score_basilica)/21)
    knn_std.append(np.std(knn_test_score_basilica))
knn_test_score_basilica
def best_kneighbor(knn_mean,knn_std):
    temp_idx = np.argmax(knn_mean)
    mean_std = np.mean(knn_std)
    while True:
        if len(knn_mean) < 1: break
        max_idx = np.argmax(knn_mean)
        max_std = knn_std[max_idx]
        if max_std <= mean_std:
            temp_idx = max_idx
            break
        del knn_mean[max_idx]
        del knn_std[max_idx]
    return temp_idx
best_kneighbor(knn_search,knn_std)
np.argmax(knn_search)
np.mean(knn_std)
knn_std[3]
knn_search[3]
np.median(knn_std)
plt.plot(knn_std)
plt.plot(range(40),knn_search)
np.mean(knn_search)
knn_search<jupyter_output><empty_output><jupyter_text>Try backward<jupyter_code>!ls {path}/models
data_lm_bwd = load_data(path/'models','data_lm_large_bwd.pkl',bs=64,backwards=True)
data_lm_bwd.vocab.stoi['this']
data_clas_bwd = (TextList
                 .from_df(df_train, path=path, vocab=data_lm_bwd.vocab, cols='Body')
                 .split_by_rand_pct(0.1, seed=42)
                 .label_from_df(cols='Label')
                 .databunch(bs=64, num_workers=os.cpu_count()*4, backwards=True))
data_clas_bwd.vocab.stoi['this']
learn_c_bwd = text_classifier_learner(data_clas_bwd, AWD_LSTM, drop_mult=0.5,
                                      loss_func=loss_func, metrics=[accuracy]).to_fp16()
learn_c_bwd.load_encoder('fine-tuned-enc-bwd');
learn_c_bwd.freeze_to(-1);
learn_c_bwd.lr_find()
learn_c_bwd.recorder.plot(suggestion=True)
lr=2e-2
lr *= 64/48
moms = (0.8,0.7)
learn_c_bwd.fit_one_cycle(2,lr,moms=moms)
learn_c_bwd.freeze_to(-2)
learn_c_bwd.fit_one_cycle(2, slice(lr/(2.6**4),lr), moms=(0.8,0.7))
learn_c_bwd.freeze_to(-3)
learn_c_bwd.fit_one_cycle(2, slice(lr/2/(2.6**4),lr/2), moms=(0.8,0.7))
learn_c_bwd.save('1a')
learn_c_bwd.unfreeze()
learn_c_bwd.fit_one_cycle(2, slice(lr/10/(2.6**4),lr/10), moms=(0.8,0.7))
learn_c_bwd.load('general-clasifier-bwd-0.83');
def _get_individual_data(filepath):
    df = pd.read_csv(filepath)
    if df.shape[0]*0.7 < 64: 
        bs= int(df.shape[0]*0.7)
    else: bs=64
    data = (TextList
            .from_df(df=df,path=path,cols='Body',vocab=data_lm_bwd.vocab)
            .split_by_rand_pct(0.3,seed=42)
            .label_from_df(cols='Label')
            .databunch(bs=bs,num_workers=os.cpu_count()*4,backwards=True)
           )
    return data

def _get_individual_learner(data):
    learn = text_classifier_learner(data,AWD_LSTM,drop_mult=0.5,
                                loss_func=FlattenedLoss(LabelSmoothingCrossEntropy),
                                metrics=[accuracy], #FBeta(beta=1)
                                model_dir = path/'models'
                               )
    if torch.cuda.is_available():
        learn.to_fp16()
    learn.load('general-clasifier-bwd-0.83')
    learn.unfreeze()
    #learn.freeze_to(-1)
    return learn

def run_valid(data_path):
    _data_valid = _get_individual_data(data_path)
    _learn_valid = _get_individual_learner(_data_valid)
    _learn_valid.fit_one_cycle(1,2e-2,moms=(0.8,0.7))
    return _learn_valid.recorder.metrics[0][0].item()
%%time
bwd_valid_acc = []
for file in valid_list:
    bwd_valid_acc.append(run_valid(file))
bwd_valid_acc
sum(bwd_valid_acc) / 21
bwd_valid_acc
sum(bwd_valid_acc) / 21
valid_list[0]
data_valid_1 = _get_individual_data(valid_list[5])
learn_valid_1 = _get_individual_learner(data_valid_1)
learn_valid_1.unfreeze()
learn_valid_1.fit_one_cycle(3,2e-2,moms=(0.8,0.7))
preds,y = learn_valid_1.get_preds()
preds.shape,y.shape
accuracy(preds,y)
preds.argmax(dim=1)
y<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13441967909096586, 'fraction_numerical': 0.021539614159955048, 'mean_word_length': 3.6863662960795787, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '439288', 'license': 'MIT', 'n_tokens_mistral': 6302, 'n_tokens_neox': 6084, 'n_words': 812, 'path': 'jupyter_notebooks/ulmfit_final.ipynb', 'repo_name': 'heye0507/Project-M', 'size': '16017'}