<commit_before>"""
Let's tune parameters to capture as many of the authors in the relevant field 
as possible. 
"""

import numpy 
import logging 
import sys 
import argparse 
from exp.influence2.ArnetMinerDataset import ArnetMinerDataset
from apgl.util.PathDefaults import PathDefaults

parser = argparse.ArgumentParser()
parser.add_argument('--runLDA', action='store_true')
args = parser.parse_args()

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
numpy.random.seed(21)

ks = numpy.array([100, 150, 200])
maxRelevantAuthors = numpy.array([100, 200, 500]) 
fields = ["Boosting", "Intelligent Agents", "Machine Learning", "Ontology Alignment"]

coverage1 = numpy.zeros((len(ks), len(fields), maxRelevantAuthors.shape[0]))
coverage2 = numpy.zeros((len(ks), len(fields), maxRelevantAuthors.shape[0]))
runLDA = args.runLDA
print(runLDA)

for m, k in enumerate(ks): 
    overwriteVectoriser = True
    overwriteModel = True
    
    for i, field in enumerate(fields): 
        for j, maxRelAuthors in enumerate(maxRelevantAuthors): 
            logging.debug("k=" + str(k) + " field=" + field + " maxRelAuthors=" + str(maxRelAuthors))
            dataset = ArnetMinerDataset(field)
            dataset.k = k 
            dataset.overwrite = True
            dataset.overwriteVectoriser = overwriteVectoriser
            dataset.overwriteModel = overwriteModel
            dataset.maxRelevantAuthors = maxRelAuthors
            dataset.dataFilename = dataset.dataDir + "DBLP-citation-1000000.txt"
            
            if runLDA:
                dataset.findSimilarDocumentsLDA()
            else: 
                dataset.findSimilarDocumentsLSI()
    
            graph, authorIndexer, relevantExperts = dataset.coauthorsGraph()
            expertMatches, expertsSet = dataset.matchExperts()
            
            coverage1[m, i, j] = float(len(expertMatches))/len(relevantExperts)
            coverage2[m, i, j] = float(len(expertMatches))/len(expertsSet)
            
            overwriteVectoriser = False 
            overwriteModel = False
            
            logging.debug("Coverage 2: " + str((k, field, maxRelAuthors)) + " = " + str(coverage2[m, i, j]))

for m, k in enumerate(ks): 
    print("k="+str(k))      
    print(coverage1[m, :, :])
    print(coverage2[m, :, :])

if runLDA: 
    resultsFilename = PathDefaults.getOutputDir() + "MatchingResultsLDA.npz"
else: 
    resultsFilename = PathDefaults.getOutputDir() + "MatchingResultsLSI.npz"<commit_msg>Run over more params <commit_after>"""
Let's tune parameters to capture as many of the authors in the relevant field 
as possible. 
"""

import numpy 
import logging 
import sys 
from exp.influence2.ArnetMinerDataset import ArnetMinerDataset
from apgl.util.PathDefaults import PathDefaults

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
numpy.random.seed(21)

algorithms = ["lsi", "lsi2", "lda"]
useTfidf = [True, False]
#ks = numpy.array([100, 150, 200])
ks = numpy.array([50])
#maxRelevantAuthors = numpy.array([100, 200, 500]) 
maxRelevantAuthors = numpy.array([100]) 
fields = ["Boosting", "Intelligent Agents", "Machine Learning", "Ontology Alignment"]

coverage1 = numpy.zeros((len(useTfidf), len(algorithms), len(ks), len(fields), maxRelevantAuthors.shape[0]))
coverage2 = numpy.zeros((len(useTfidf), len(algorithms), len(ks), len(fields), maxRelevantAuthors.shape[0]))

for p, tfidf in enumerate(useTfidf):
    for n, algorithm in enumerate(algorithms):
        for m, k in enumerate(ks): 
            overwriteVectoriser = True
            overwriteModel = True
                    
            for i, field in enumerate(fields): 
                for j, maxRelAuthors in enumerate(maxRelevantAuthors): 
                    logging.debug("k=" + str(k) + " field=" + field + " maxRelAuthors=" + str(maxRelAuthors) + " algorithm=" + str(algorithm) + " useTfidf=" + str(tfidf))
                    
                    dataset = ArnetMinerDataset(field)
                    dataset.k = k 
                    dataset.overwrite = True
                    dataset.overwriteVectoriser = overwriteVectoriser
                    dataset.overwriteModel = overwriteModel
                    dataset.maxRelevantAuthors = maxRelAuthors
                    dataset.dataFilename = dataset.dataDir + "DBLP-citation-100000.txt"
                    dataset.tfidf = tfidf
                    
                    if algorithm == "lsi":
                        dataset.findSimilarDocumentsLSI()
                    elif algorithm == "lsi2": 
                        dataset.findSimilarDocumentsLSI2()
                    else: 
                        dataset.findSimilarDocumentsLDA()
            
                    graph, authorIndexer, relevantExperts = dataset.coauthorsGraph()
                    expertMatches, expertsSet = dataset.matchExperts()
                    
                    if len(relevantExperts) != 0: 
                        coverage1[p, n, m, i, j] = float(len(expertMatches))/len(relevantExperts)
                    if len(expertsSet) != 0: 
                        coverage2[p, n, m, i, j] = float(len(expertMatches))/len(expertsSet)
                    
                    overwriteVectoriser = False 
                    overwriteModel = False
                    
                    logging.debug("Coverage 2: " + str((tfidf, algorithm, k, field, maxRelAuthors)) + " = " + str(coverage2[p, n, m, i, j]))

meanCoverage1 = numpy.mean(coverage1, 3)
meanCoverage2 = numpy.mean(coverage2, 3)

for p, tfidf in enumerate(useTfidf):
    for n, algorithm in enumerate(algorithms):
        for m, k in enumerate(ks): 
            logging.debug(str((tfidf, algorithm, k)) + " " + str(meanCoverage2[p, n, m, :]))


resultsFilename = PathDefaults.getOutputDir() + "MatchingResults.npz"
{'fraction_non_alphanumeric': 0.11055536391859262, 'fraction_numerical': 0.015522593997930321, 'mean_word_length': 2.4395017793594307, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4599660', 'n_tokens_mistral': 1840, 'n_tokens_neox': 1659, 'n_words': 371}