<jupyter_start><jupyter_text>Data From NIPS importation<jupyter_code>from pathlib import Path 

df = pd.read_json("nips/papers.json")
df.head()<jupyter_output><empty_output><jupyter_text>Data From ACL importation<jupyter_code>import pandas as pd

data = pd.read_csv('2014/paper_ids.txt', sep="\t", header=None)
data.columns = ['id','paper','year']
ids = data.id
from pathlib import Path
papers_ACL = []
for el in ids:
    name = Path('2014/papers_text/'+el+'.txt')
    if name.is_file():   
        with open(name, 'r') as myfile:
            data = myfile.read().replace('\n', '')
            papers_ACL.append([el,data])<jupyter_output><empty_output><jupyter_text>Cleaning function<jupyter_code>def cleaning(paper):
    # split into words
    from nltk.tokenize import word_tokenize
    tokens = word_tokenize(paper)
    # convert to lower case
    tokens = [w.lower() for w in tokens]
    # remove punctuation from each word
    import string
    table = str.maketrans('', '', string.punctuation)
    stripped = [w.translate(table) for w in tokens]
    # remove remaining tokens that are not alphabetic
    words = [word for word in stripped if word.isalpha()]
    # filter out stop words
    from nltk.corpus import stopwords
    stop_words = set(stopwords.words('english'))
    words = [w for w in words if not w in stop_words]
    # stemming of words
    from nltk.stem import WordNetLemmatizer
    wordnet_lemmatizer = WordNetLemmatizer()
    lematized = [wordnet_lemmatizer.lemmatize(word) for word in words]
    lematized2 = [wordnet_lemmatizer.lemmatize(word,'v') for word in words]
    # remove words of less than 3 characters
    for word in lematized2:
        if(len(word)<3):
            lematized2.remove(word)  
    return(lematized2)<jupyter_output><empty_output><jupyter_text>Papers from NIPS ready for TFIDF<jupyter_code>papers_NIPS_cleaned=[]
papers_NIPS = df.paper_text
for paper in papers_NIPS:
    papers_NIPS_cleaned.append(cleaning(paper))
papers_cleaned=[]
for i in range(0,len(papers_NIPS_cleaned)):
    papers_cleaned.append(' '.join(papers_NIPS_cleaned[i]))<jupyter_output><empty_output><jupyter_text>Papers from ACL ready for TFIDF after joining papers_NIPS<jupyter_code>papers_ACL_cleaned=[]
for i in range(0,len(papers_ACL)):
    papers_ACL_cleaned.append(cleaning(papers_ACL[i][1]))
len(papers_ACL_cleaned)
len(papers_cleaned)
for i in range(0,len(papers_ACL_cleaned)):
    papers_cleaned.append(' '.join(papers_ACL_cleaned[i]))
len(papers_cleaned)<jupyter_output><empty_output><jupyter_text>Export and import cleaned papers<jupyter_code>import pickle

with open('papers_cleaned.txt', 'wb') as fp:
    pickle.dump(papers_cleaned, fp)
with open ('papers_cleaned.txt', 'rb') as fp:
    papers_cleaned_load = pickle.load(fp)<jupyter_output><empty_output><jupyter_text>TFIDF and NMF<jupyter_code>#set parameters 
n_features = 100 #features to use in the tifidf transformation 
n_topics = 8  #number of topics that will be displayed  ###RELEVANT PARAMETER 
n_top_words = 10 #number of words composing each topic 
#tfidf with n_features, removing stop words for English 
tfidf_vectorizer = TfidfVectorizer(max_df=0.95, min_df=0.2,max_features=n_features,stop_words='english')
#fit tdidf with the paper_text
tfidf = tfidf_vectorizer.fit_transform(papers_cleaned_load)
#transforming the tfidf representation into NMF matrix descomposition with n_topics as parameter
nmf = NMF(n_components=n_topics, random_state=0,alpha=.1, l1_ratio=.5).fit(tfidf)
##result in a 8*100 matrix
##Funtion to retrieve the n_top_words from the transformation 
##(taken from https://www.kaggle.com/dschniertshauer)
def print_top_words(model, feature_names, n_top_words):
    for topic_idx, topic in enumerate(model.components_):
        print("\nTopic #%d:" % topic_idx)
        print(" ".join([feature_names[i]
                        for i in topic.argsort()[:-n_top_words - 1:-1]]))
    print()<jupyter_output><empty_output><jupyter_text>TopicsTopic 0: Information ExtractionTopic 1: Stochastic MethodsTopic 2: Parsing TechniquesTopic 3: Probabilistic MethodsTopic 4: Reinforcement LearningTopic 5: Translation TechniquesTopic 6: Words SegmentationTopic 7: Training Neural Networks<jupyter_code>##Printing the topics 
print("Topics found via NMF:")
tfidf_feature_names = tfidf_vectorizer.get_feature_names()
print_top_words(nmf, tfidf_feature_names, n_top_words)
#Finding relation between topics and papers
nmf_embedding = nmf.transform(tfidf)
nmf_embedding = (nmf_embedding - nmf_embedding.mean(axis=0))/nmf_embedding.std(axis=0)

##Getting the titles of the most related papers to the topics, in the case of ACL the ids of the papers 
##so the decision of the topic name, could be less subjective
top_idx = np.argsort(nmf_embedding,axis=0)[-5:]
##8 topics
count = 0
for idxs in top_idx.T: 
    print("\nTopic {}:".format(count))
    for idx in idxs:
        if(idx>6559):
            print(papers_ACL[idx-6560][0])
        else:
            print(df.iloc[idx]['title'])
    count += 1<jupyter_output>Topic 0:
A00-1025
P95-1042
Q14-1012
J02-4005
W14-5210

Topic 1:
Stochastic Variance Reduction Methods for Saddle-Point Problems
Random Walk Approach to Regret Minimization
Stochastic Online AUC Maximization
Bayesian Optimization with Exponential Convergence
Fitted Q-iteration in continuous action-space MDPs

Topic 2:
P83-1018
N03-2011
I05-1013
J83-2002
E89-1003

Topic 3:
J88-3012
Pairwise Choice Markov Chains
Comparing Bayesian models for multisensory cue combination without mandatory integration
W02-0505
N03-2038

Topic 4:
E06-3008
Do Convnets Learn Correspondence?
A00-2028
W11-0313
N09-3010

Topic 5:
W02-0701
P09-4005
N04-4003
C14-1192
N10-1078

Topic 6:
C94-1049
W10-3606
W97-0120
W10-3212
C96-1035

Topic 7:
Learning in Networks of Nondeterministic Adaptive Logic Elements
Dynamically-Adaptive Winner-Take-All Networks
Skeletonization: A Technique for Trimming the Fat from a Network via Relevance Assessment
Learning long-term dependencies is not as difficult with NARX networks
Backpro[...]<jupyter_text>Saving classification of papers<jupyter_code>text_file=open("paperSource_paperID_paperTopic.csv","w")
text_file.write("%s,%s,%s\n"%("source","paper_id","topic_id"))
j=0
for i in range(len(papers_cleaned_load)):
    if(i<6560):
        text_file.write("NIPS,%s,%s \n"%(df.iloc[i].id,nmf_embedding.argmax(axis=1)[i]))
        #print("NIPS",",",df.iloc[i].id, ",",nmf_embedding.argmax(axis=1)[i])
    else:
        text_file.write("ALC,%s,%s \n"%(papers_ACL[j][0],nmf_embedding.argmax(axis=1)[i]))
        #print("ALC",",",papers_ACL[j][0],",",nmf_embedding.argmax(axis=1)[i])
        j+=1
text_file.close()    
papers_topics = pd.read_csv('paperSource_paperID_paperTopic.csv')
papers_authors_NIPS = pd.read_csv('Nips/paper_authors.csv')
papers_authors_ACL = pd.read_csv('2014/paper_author_affiliations.txt',sep='\t')
del papers_authors_ACL['affiliation id']<jupyter_output><empty_output><jupyter_text>Creation of a dictionary of authors and their topics<jupyter_code>#paper_authors into a dict
papers_authors_NIPS_transform = papers_authors_NIPS.groupby('paper_id')['author_id'].apply(list)
papers_authors_ACL_transform = papers_authors_ACL.groupby('paper id')['author id'].apply(list)
authors_topics_NIPS_todos = {}
authors_topics_ACL_todos = {}
keyCountError = 0
for i in range(0,len(papers_topics)):
        if(papers_topics.iloc[i]['source']=="NIPS"):
            try:
                authors_paper = papers_authors_NIPS_transform[int(papers_topics.iloc[i]['paper_id'])]
                for el in authors_paper:
                    if(el in authors_topics_NIPS_todos):
                        authors_topics_NIPS_todos[el]+=[papers_topics.iloc[i]['topic_id']]
                    else:
                        authors_topics_NIPS_todos[el] = [papers_topics.iloc[i]['topic_id']]
            except KeyError:
                keyCountError+=1
        else:
            try:
                authors_paper = papers_authors_ACL_transform[papers_topics.iloc[i]['paper_id']]
                for el in authors_paper:
                    if(el in authors_topics_ACL_todos):
                        authors_topics_ACL_todos[el]+=[papers_topics.iloc[i]['topic_id']]
                    else:
                        authors_topics_ACL_todos[el] = [papers_topics.iloc[i]['topic_id']]
            except KeyError:
                keyCountError+=1<jupyter_output><empty_output><jupyter_text>Keep most common topic for each author<jupyter_code>def most_common(lst):
    return max(set(lst), key=lst.count)
authors_topics_NIPS_mc = {}
for el in authors_topics_NIPS_todos:
    authors_topics_NIPS_mc[el] = most_common(authors_topics_NIPS_todos[el])
authors_topics_ACL_mc = {}
for el in authors_topics_ACL_todos:
    authors_topics_ACL_mc[el] = most_common(authors_topics_ACL_todos[el])<jupyter_output><empty_output><jupyter_text>Authors per topic per source<jupyter_code>topics_authors_NIPS = {}
for i in range(0,n_topics):
    topics_authors_NIPS[i] = []
for el in authors_topics_NIPS_mc:
    topics_authors_NIPS[authors_topics_NIPS_mc[el]]+=[el]
topics_authors_ACL = {}
for i in range(0,n_topics):
    topics_authors_ACL[i] = []
for el in authors_topics_ACL_mc:
    topics_authors_ACL[authors_topics_ACL_mc[el]]+=[el]<jupyter_output><empty_output><jupyter_text>Export and import authors per topic per source<jupyter_code>import pickle

with open('topics_authors_NIPS.txt', 'wb') as fp:
    pickle.dump(topics_authors_NIPS, fp)
    
with open('topics_authors_ACL.txt', 'wb') as fp:
    pickle.dump(topics_authors_ACL, fp)
import pickle
with open ('topics_authors_NIPS.txt', 'rb') as fp:
    topics_authors_NIPS = pickle.load(fp)
    
with open ('topics_authors_ACL.txt', 'rb') as fp:
    topics_authors_ACL = pickle.load(fp)<jupyter_output><empty_output><jupyter_text>Load names<jupyter_code>authors_ACL = pd.read_csv('2014/author_ids.txt',sep='\t',header=None)
authors_ACL.columns = ['author_id','author_name']
authors_NIPS = pd.read_csv('Nips/authors.csv')
authors_NIPS.columns = ['author_id','author_name']
#authors_id into a dict
authors_ACL_dict = authors_ACL.groupby('author_id')['author_name'].apply(list)
authors_NIPS_dict = authors_NIPS.groupby('author_id')['author_name'].apply(list)<jupyter_output><empty_output><jupyter_text>Given an author name from NIPS, return similar authors from ACL<jupyter_code>def NIPS_to_ACL(author_name):
    for i in range(0,len(authors_NIPS)):
        if(authors_NIPS.iloc[i]['author_name']==author_name):
            author_id = authors_NIPS.iloc[i]['author_id']
    for i in range(0,n_topics):
        if(author_id in topics_authors_NIPS[i]):
            topic_id = i
            break
    ret = []
    for el in topics_authors_ACL[topic_id]:
        ret.append(authors_ACL_dict[el])
    return ret
NIPS_to_ACL('<NAME>')[0]<jupyter_output><empty_output><jupyter_text>Given an author name from ACL, return similar authors from NIPS<jupyter_code>def ACL_to_NIPS(author_name):
    for i in range(0,len(authors_ACL)):
        if(authors_ACL.iloc[i]['author_name']==author_name):
            author_id = authors_ACL.iloc[i]['author_id']
    for i in range(0,n_topics):
        if(author_id in topics_authors_ACL[i]):
            topic_id = i
            break
    ret = []
    for el in topics_authors_NIPS[topic_id]:
        ret.append(authors_NIPS_dict[el])
    return ret
ACL_to_NIPS('Szpakowicz,Stan')[0]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11223324844120489, 'fraction_numerical': 0.023272152454553437, 'mean_word_length': 4.7631578947368425, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 66, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '403694', 'license': 'MIT', 'n_tokens_mistral': 4373, 'n_tokens_neox': 4058, 'n_words': 808, 'path': 'Similarities between authors from NIPS and ACL using NMF Topic Modeling.ipynb', 'repo_name': 'luisgaldo/Similarities-between-researchers', 'size': '11395'}