<commit_before>
import numpy 
import matplotlib 
matplotlib.use("GTK3Agg")
import matplotlib.pyplot as plt 
from exp.influence2.ArnetMinerDataset import ArnetMinerDataset
from apgl.util.Latex import Latex 
from apgl.util.Util import Util 
from apgl.util.Evaluator import Evaluator 

numpy.set_printoptions(suppress=True, precision=3, linewidth=100)
dataset = ArnetMinerDataset()

ns = numpy.arange(5, 55, 5)
averagePrecisionN = 20 
bestPrecisions = numpy.zeros((len(ns), len(dataset.fields)))
bestAveragePrecisions = numpy.zeros(len(dataset.fields))

for i, field in enumerate(dataset.fields): 
    outputFilename = dataset.getResultsDir(field) + "outputLists.npz"
    outputLists, expertMatchesInds = Util.loadPickle(outputFilename)
    
    numMethods = len(outputLists)
    precisions = numpy.zeros((len(ns), numMethods))
    averagePrecisions = numpy.zeros(numMethods)
    
    for i, n in enumerate(ns):     
        for j in range(len(outputLists)): 
            precisions[i, j] = Evaluator.precisionFromIndLists(expertMatchesInds, outputLists[j][0:n]) 
        
    for j in range(len(outputLists)):                 
        averagePrecisions[j] = Evaluator.averagePrecisionFromLists(expertMatchesInds, outputLists[j][0:averagePrecisionN], averagePrecisionN) 
    
    print(field)
    print(precisions)
    print(averagePrecisions)
    
    bestInd = numpy.argmax(averagePrecisions)
    plt.plot(ns, precisions[:, bestInd], label=field)
    bestPrecisions[:, i] = precisions[:, bestInd]
    bestAveragePrecisions[i] = averagePrecisions[bestInd]

bestPrecisions2 = numpy.c_[numpy.array(ns), bestPrecisions]
print(Latex.array2DToRows(bestPrecisions2))
print(Latex.array1DToRow(bestAveragePrecisions))

print(dataset.fields)

plt.legend()
plt.show()<commit_msg>Print coverages and deal with missing files <commit_after>
import numpy 
import matplotlib 
matplotlib.use("GTK3Agg")
import matplotlib.pyplot as plt 
from exp.influence2.ArnetMinerDataset import ArnetMinerDataset
from apgl.util.Latex import Latex 
from apgl.util.Util import Util 
from apgl.util.Evaluator import Evaluator 

ranLSI = False
numpy.set_printoptions(suppress=True, precision=3, linewidth=100)
dataset = ArnetMinerDataset(runLSI=ranLSI)

ns = numpy.arange(5, 55, 5)
averagePrecisionN = 30 
bestPrecisions = numpy.zeros((len(ns), len(dataset.fields)))
bestAveragePrecisions = numpy.zeros(len(dataset.fields))

coverages = numpy.load(dataset.coverageFilename)
print("==== Coverages ====")
print(coverages)

for s, field in enumerate(dataset.fields): 
    if ranLSI: 
        outputFilename = dataset.getOutputFieldDir(field) + "outputListsLSI.npz"
    else: 
        outputFilename = dataset.getOutputFieldDir(field) + "outputListsLDA.npz"
        
    try: 
        outputLists, expertMatchesInds = Util.loadPickle(outputFilename)
        
        
        numMethods = len(outputLists)
        precisions = numpy.zeros((len(ns), numMethods))
        averagePrecisions = numpy.zeros(numMethods)
        
        for i, n in enumerate(ns):     
            for j in range(len(outputLists)): 
                precisions[i, j] = Evaluator.precisionFromIndLists(expertMatchesInds, outputLists[j][0:n]) 
            
        for j in range(len(outputLists)):                 
            averagePrecisions[j] = Evaluator.averagePrecisionFromLists(expertMatchesInds, outputLists[j][0:averagePrecisionN], averagePrecisionN) 
        
        print(field)
        print(precisions)
        print(averagePrecisions)
        
        bestInd = numpy.argmax(averagePrecisions)
        plt.plot(ns, precisions[:, bestInd], label=field)
        bestPrecisions[:, s] = precisions[:, bestInd]
        bestAveragePrecisions[s] = averagePrecisions[bestInd]
    except IOError as e: 
        print(e)

bestPrecisions2 = numpy.c_[numpy.array(ns), bestPrecisions]
print(Latex.array2DToRows(bestPrecisions2))
print(Latex.array1DToRow(bestAveragePrecisions))

print(dataset.fields)

plt.legend()
plt.show()
{'fraction_non_alphanumeric': 0.1042139793086046, 'fraction_numerical': 0.009084027252081756, 'mean_word_length': 4.024081115335868, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1985701', 'n_tokens_mistral': 1378, 'n_tokens_neox': 1267, 'n_words': 233}