<jupyter_start><jupyter_text>Notebook for Authorship Attribution using multichanels<jupyter_code>#python basic libs
import os;
from os.path import join as pathjoin;
import zipfile;

import re;
import random;
import json;
from collections import defaultdict, Counter;
from pprint import pprint
from time import time
%matplotlib inline
#data analysis libs
import numpy as np;
import pandas as pd;
import matplotlib.pyplot as plt;
import seaborn as sns;
#machine learning libs
#feature extraction
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, TfidfTransformer
from sklearn import feature_selection;

#preprocessing and transformation
from sklearn import preprocessing;
from sklearn import decomposition;

from sklearn.base import BaseEstimator, ClassifierMixin

#classifiers
from sklearn import linear_model
from sklearn import neural_network
from sklearn import svm
from sklearn import ensemble
from sklearn import tree
from sklearn import metrics

#
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline

#model valuation
from sklearn.metrics import f1_score, precision_score, recall_score, accuracy_score;
#import spacy;
import gensim
import logging
from gensim.models import KeyedVectors
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.CRITICAL)
from IPython.display import Markdown, display, HTML
import warnings
from sklearn.exceptions import UndefinedMetricWarning
warnings.simplefilter(action='ignore', category=UndefinedMetricWarning)
warnings.simplefilter(action='ignore', category=FutureWarning)
np.set_printoptions(precision=4)
pd.options.display.float_format = '{:,.4f}'.format
sns.set(color_codes=True);
import platform;
import sklearn;
import scipy;

Markdown(
    "|%-15s|%-65s|\n"%("PACK","VERSION") +
    "|%-15s|%-65s|\n"%('-'*15,'-'*65) +
    ('\n'.join(
    "|%-15s|%-45s| " % (pack, version.strip())
    for pack, version in
    zip(['SO','NumPy','SciPy','Scikit-Learn','Seaborn','Gensim'],
        [platform.platform(), np.__version__, scipy.__version__,
         sklearn.__version__, sns.__version__,  gensim.__version__])

)))
baseDir = '../data/';
corpusTraining    = 'pan18-cross-domain-authorship-attribution-training-dataset-2017-12-02';
corpusEvaluation = 'pan18-cross-domain-authorship-attribution-test-dataset2-2018-04-20';
corpusEach1 = 'AvaliacaoPT';

currentCorpus = corpusEvaluation;

inputDir= pathjoin(baseDir,currentCorpus);
outputDir= pathjoin(baseDir,'out');
if not os.path.exists(outputDir):
    os.mkdir(outputDir);
import pan
import sklearnExtensions
from sklearnExtensions import DenseTransformer
problems = pan.readCollectionsOfProblems2019(inputDir);
pd.DataFrame(problems)
pd.DataFrame(problems[0]['unknown'],columns=['text','label','file']).head(10)
pd.DataFrame(problems).groupby(['language']).count().reset_index()
def tag(p):
    df = pd.concat([
            pd.DataFrame(p['candidates']),
            pd.DataFrame(p['unknown'])]
    );
    df.columns=['text','candidate','fandom'];
    df['text_char_length'] = df['text'].apply(lambda x:len(x))
    df['text_word_length'] = df['text'].apply(lambda x:len(re.findall(r'\b\w+\b',x)))

    #df.drop(columns=['fandom'],inplace=True)
    df['set'] = ['train']*len(p['candidates']) + ['val']*len(p['unknown'])
    df['problem-name'] = p['problem-name'];
    df['language'] = p['language'];
    return df;
df = pd.concat([tag(p) for p in problems])

p1 = pd.pivot_table(df,index=['problem-name'], columns='set', values=['text_char_length','text_word_length'],aggfunc='mean')

def a(x):
    c= x[['text','candidate']].groupby('candidate').count().reset_index();
    
    return pd.Series({
     'docs_per_author': c['text'].mean()
    })
    
p2 = df.groupby(['problem-name','set']).apply(a).reset_index().pivot(index='problem-name',columns='set')

corpus_stats = pd.merge(p1,p2,on='problem-name')
def readBaseline():
    baselines = ['svm_evaluation'];
    baselineResults  = [json.load(open('./outputEvaluation/'+f+'/out.json')) for f in baselines];
    
    #converting the overall into a row
    baselineResults2 = []
    for p,b in zip(baselineResults,baselines):
        t = p['problems'];
        t.append({'problem-name':'overall', 'macro-f1':p['overall_score']})
        for tt in t:
            tt['method'] = b;
        baselineResults2 +=t;
    
    baselineResults2 = pd.DataFrame(baselineResults2);
    return baselineResults2;

baselineResults = readBaseline()

p3 = baselineResults[['problem-name','method','macro-f1']].pivot(index='problem-name',columns='method')
pd.merge(corpus_stats,p3,on='problem-name')\
    .style.background_gradient(cmap=sns.light_palette("gray", as_cmap=True),axis=0)
def a():
    df = pd.merge(
        pd.DataFrame(baselineResults),
        pd.DataFrame(problems)[['problem-name','language']],
        on='problem-name',
        how='left'
    );
    df['problem-name'] = df['problem-name'].apply(lambda x: x.replace('problem000','p'))
    plt.figure(figsize=(18,5))
    sns.barplot(data=df, x='problem-name',y='macro-f1', hue='language');
    plt.legend(loc=1);
    #plt.title("Macro-F1 score")
a();
def a():
    df = pd.DataFrame(baselineResults);
    df['problem-name'] = df['problem-name'].apply(lambda x: x.replace('problem000',''))
    plt.figure(figsize=(12,2.5))
    sns.boxplot(data=pd.DataFrame(baselineResults),x='macro-f1', y='method',whis="range", palette="viridis");
a();
def upsample_text(texts, labels, min_doc_sample = 10):
    texts = texts.copy();
    labels = labels.copy();
    counter = Counter(labels);
    doc_per_label = max(max(counter.values()),min_doc_sample);
    
    new_texts  = [];
    new_labels = [];
    
    for label, cnt in counter.items():
        temp_texts = [t for t,l in zip(texts,labels) if l == label];
        
        while cnt < doc_per_label:
            texts_sizes = np.array([len(t) for t in temp_texts]);
            #in descending order;
            temp_texts = np.array(temp_texts)[np.argsort(-texts_sizes)].tolist();
            
            text = temp_texts.pop(0);
            l = int(0.25*len(text));
            temp_texts.append(text[:(len(text)-l)])
            temp_texts.append(text[l:]);
            
            cnt = len(temp_texts)
            
        
        new_texts  += temp_texts;
        new_labels += [label] * cnt;
        
    
    indexes = np.arange(len(new_texts));
    np.random.shuffle(indexes);
        
    return np.array(new_texts)[indexes].tolist(), np.array(new_labels)[indexes].tolist()
    
#train_docs, train_labels = upsample_text(train_docs, train_labels);
def eval_measures(test_labels, predictions):
    gt = {i:label for i,label in enumerate(test_labels) }
    pred = {i:label for i,label in enumerate(predictions) }
    
    ## keep pan original eval code
    actual_authors = list(gt.values())
    encoder = preprocessing.LabelEncoder().fit(['<UNK>'] + actual_authors)

    text_ids, gold_authors, silver_authors = [], [], []
    for text_id in sorted(gt):
        text_ids.append(text_id)
        gold_authors.append(gt[text_id])
        try:
            silver_authors.append(pred[text_id])
        except KeyError:
            # missing attributions get <UNK>:
            silver_authors.append('<UNK>')


    # replace non-existent silver authors with '<UNK>':
    silver_authors = [a if a in encoder.classes_ else '<UNK>' 
                      for a in silver_authors]

    gold_author_ints = encoder.transform(gold_authors)
    silver_author_ints = encoder.transform(silver_authors)

    # get F1 for individual classes (and suppress warnings):
    with warnings.catch_warnings():
        warnings.simplefilter('ignore')
        labels=list(set(gold_author_ints))
        # Exclude the <UNK> class
        f1 = metrics.f1_score(gold_author_ints, silver_author_ints,labels,average='macro')
        precision = metrics.precision_score(gold_author_ints, silver_author_ints,labels,average='macro')
        recall = metrics.recall_score(gold_author_ints, silver_author_ints,labels,average='macro')
        accuracy = metrics.accuracy_score(gold_author_ints,silver_author_ints)

    return f1,precision,recall,accuracy<jupyter_output><empty_output><jupyter_text>Main Logic<jupyter_code>def runModel(problem, pipeline, parameters, probCut=0.055,upsample=0, n_jobs=-1,verbose=False):
    print ("\nProblem: %s,  language: %s, " %(problem['problem-name'],problem['language']), end=' ')
    
    train_docs, train_labels, _ = zip(*problem['candidates'])
    test_docs, test_labels, _   = zip(*problem['unknown'])
    
    if upsample> 0:
        train_docs, train_labels = upsample_text(list(train_docs), list(train_labels),min_doc_sample=upsample);
    
    grid_search = GridSearchCV(pipeline,
                               parameters,cv=3,
                               iid=False, n_jobs=n_jobs, verbose=verbose, scoring='f1_macro')
    
    t0 = time()
    grid_search.fit(train_docs, train_labels);
    grid_search.best_estimator_.fit(train_docs, train_labels)
    
    print("Gridsearh %0.3fs" % (time() - t0))

    test_pred      =grid_search.predict(test_docs);
    test_pred_proba=grid_search.predict_proba(test_docs);
    dist = test_pred_proba.std(axis=1);
    
    f1,precision,recall,accuracy = eval_measures(test_labels,test_pred)
    report = metrics.classification_report(test_labels,test_pred, output_dict=True);
    print("Best score: %0.3f - Test score: %0.3f - in %0.3fs" % (grid_search.best_score_, f1, time() - t0))
    best_parameters = grid_search.best_estimator_.get_params()
    print("'"+problem['language']+"':{")
    for param_name in sorted(parameters.keys()):
        print("\t'%s': %r," % (param_name, best_parameters[param_name]))
    print("},")
    
    return {
                'problem-name'  : problem['problem-name'],
                'f1'       : round(f1,4),
                'accuracy' : round(accuracy,4),
                'report'   : report,
                'best_cut' : round(distmax,4)
                
        }, grid_search.cv_results_,best_parameters;
def runGrid(funct):
    result = [];
    cv_result = [];
    best_parameters = [];
    for problem in problems:
        r, c, b = funct(problem);
        result.append(r);
        cv_result.append(c);
        b['problem-name'] = problem['problem-name'];
        best_parameters.append(b);
    return {'result':result,'cv':cv_result,'best_params':best_parameters};
allResults = {};
def runCharModel(problem):
    pipeline = Pipeline([
        ('vect',TfidfVectorizer(analyzer='char',
                ngram_range=(2,5),
                min_df=0.01,
                lowercase=False,
                smooth_idf=True,
                sublinear_tf=True,
                norm='l1',
                dtype=np.float32,
                )),
        ('scaler', preprocessing.MaxAbsScaler()),
        #('selection',feature_selection.SelectPercentile(percentile=90)),
        #('dense',DenseTransformer()),
        ('pca'  ,decomposition.TruncatedSVD(100)),
        ('clf', linear_model.LogisticRegression(random_state=0,C=1,multi_class='multinomial', solver='newton-cg')),
    ])
    
    # uncommenting more parameters will give better exploring power but will
    # increase processing time in a combinatorial way
    parameters = {
        'vect__ngram_range':((2,4),(2,5)),
        'vect__sublinear_tf':(True,False),
    }
    return runModel(problem,pipeline, parameters,upsample=0,n_jobs=-1,verbose=True)
allResults['char'] =runGrid(runCharModel);
pd.DataFrame(allResults['char']['result'])['f1'].plot(kind='bar')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14731294280283394, 'fraction_numerical': 0.012009676861931917, 'mean_word_length': 3.781082197439075, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '424154', 'license': 'Apache-2.0', 'n_tokens_mistral': 3936, 'n_tokens_neox': 3742, 'n_words': 659, 'path': 'Article2019/Article 2019.ipynb', 'repo_name': 'jeleandro/PANAA2018', 'size': '11574'}