<commit_before>
"""
We will analyse the article metadata file and extract co-authors. 
"""

from apgl.util.PathDefaults import PathDefaults 
import logging 
import sys 
import itertools 
import igraph 
import matplotlib.pyplot as plt 
import json 

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)

#path = "/local/dhanjalc/dataDump-28-11-12/" 
path = PathDefaults.getDataDir() + "erasm/"

#fileName = path + "articleMetadata-28-11-12"
fileName = path + "articleMetadata100000"


fileObj = open(fileName, 'r')
articleMetadataList = []
vertexIdDict = {} 
vertexIdSet = set([])
vertexIdList = []
edgeSet = set([])
edgeArray = []

i = 0 

for line in fileObj: 
    if i % 1000 == 0: 
        print(i)
    
    articleMetaData = json.loads(line)
    
    if "authors" in articleMetaData: 
        authors = articleMetaData["authors"]
        
        coauthorList = []
        for author in authors: 
            authorString = author["forename"] + " " + author["surname"]
            authorString = authorString.strip()         
            
            if len(authorString) != 0: 
                coauthorList.append(authorString)
        
            if len(authorString) != 0 and authorString not in vertexIdSet:  
                vertexIdDict[authorString] = i
                vertexIdSet.add(authorString)
                vertexIdList.append(authorString)
                i += 1 
        
        iterator = itertools.permutations(range(len(coauthorList)), 2)
        
        for j, k in iterator: 
            vId1 = coauthorList[j]
            vId2 = coauthorList[k]            
            
            if (vertexIdDict[vId1], vertexIdDict[vId2]) not in edgeSet and (vertexIdDict[vId2], vertexIdDict[vId1]) not in edgeSet: 
                edgeArray.append([vertexIdDict[vId1], vertexIdDict[vId2]])
                edgeSet.add((vertexIdDict[vId1], vertexIdDict[vId2]))

graph = igraph.Graph()
graph.add_vertices(len(vertexIdSet))
graph.add_edges(edgeArray) 

print(graph.summary())

sortedNames = sorted(vertexIdDict.keys(), key=lambda name: name.split()[-1])
#for name in sortedNames: 
#    print(name)

graphFileName = path + "coauthorsGraph" 
graph.save(graphFileName, "edgelist")

logging.debug("Saved graph as " + graphFileName)<commit_msg>Output igraph directly <commit_after>
"""
We will analyse the article metadata file and extract co-authors. 
"""

from apgl.util.PathDefaults import PathDefaults 
import os
import logging 
import sys 
import itertools 
import igraph 
import matplotlib.pyplot as plt 
import json 
import gc 

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)

path = "/local/dhanjalc/dataDump-28-11-12/" 
fileName = path + "articleMetadata-28-11-12"

if not os.path.exists(fileName): 
    path = PathDefaults.getDataDir() + "erasm/"
    fileName = path + "articleMetadata100000"

logging.debug("Loading article metadata from " + fileName)

fileObj = open(fileName, 'r')
vertexIdDict = {} 
vertexIdSet = set([])
vertexIdList = []
edgeSet = set([])
edgeArray = []

i = 0 
lineInd = 0 
emptyAuthors = 0

newEdges = []
graph = igraph.Graph()

#TODO: Write out file in real time 

for line in fileObj: 
    if lineInd % 1000 == 0: 
        print("Line " + str(lineInd) + " Author " + str(len(vertexIdSet)) + " empty author strings " + str(emptyAuthors)) 
        graph.add_edges(newEdges)
        del newEdges 
        newEdges = []
        gc.collect()
    
    articleMetaData = json.loads(line)
    
    if "authors" in articleMetaData: 
        authors = articleMetaData["authors"]
        del articleMetaData
        
        coauthorList = []
        for author in authors: 
            authorString = "".join([author["forename"], " ", author["surname"]])
            authorString = authorString.strip()         
            
            if len(authorString) != 0: 
                if authorString not in vertexIdSet: 
                    graph.add_vertex(authorString)                   
                
                coauthorList.append(authorString)
                vertexIdSet.add(authorString)
                
                del authorString 
            else: 
                emptyAuthors += 1
            
        iterator = itertools.combinations(coauthorList, 2)
        del coauthorList 
        
        for vId1, vId2 in iterator:         
            newEdges.append([vId1, vId2])

    lineInd += 1 

print(sys.getsizeof(vertexIdDict))
print(sys.getsizeof(vertexIdSet))
print(sys.getsizeof(vertexIdList))
print(sys.getsizeof(edgeSet))
print(sys.getsizeof(edgeArray))
print(sys.getsizeof(newEdges))
print(sys.getsizeof(graph))

logging.debug(graph.summary())

print(graph.neighbors("<NAME>"))

#sortedNames = sorted(vertexIdDict.keys(), key=lambda name: name.split()[-1])
#for name in sortedNames: 
#    print(name)

graphFileName = path + "coauthorsGraph" 
graph.save(graphFileName, "edgelist")

logging.debug("Saved graph as " + graphFileName)
{'fraction_non_alphanumeric': 0.11079487700752186, 'fraction_numerical': 0.015043708070746087, 'mean_word_length': 2.5991221653255305, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3200376', 'n_tokens_mistral': 1572, 'n_tokens_neox': 1486, 'n_words': 333}