<jupyter_start><jupyter_text>Count paper numbers<jupyter_code>paper_data = {}
def count_paper_numbers(venue):
    paper_data[venue] = {y:0 for y in yearranges}
    for year in yearranges:
        with open(filtered_papers_filename(venue, year), "r") as fh:
            papers = json.load(fh)
            paper_data[venue][year] = len(papers)
    paper_data[venue]["Total"] = sum(paper_data[venue].values())
with open(venue_category_filename, "r") as fh:
    reader = csv.reader(fh, delimiter=",")
    # skip header row
    next(reader)
    for row in reader:
        venue_name = row[0]
        count_paper_numbers(venue_name)<jupyter_output><empty_output><jupyter_text>Compute Geometric meanOne difficulty with the geometric mean is that it is undefined when the number of citations is zero. There indeed are some papers with zero citations. To handle this case, we add 1 to the number of citations before computing the geometric mean, and then subtract 1 from the mean obtained.<jupyter_code>def compute_venue_weight(venue):
    citation_numbers = []
    paper_numbers = []
    no_matching_paper_count =0
#     for year in yearranges:
    for year in yearranges:
        with open(filtered_papers_filename(venue, year), "r") as fh:
            papers = json.load(fh)
            paper_numbers.append(len(papers))
            for p in papers:
                mag_matching_papers = p["MAG papers"]
                if len(mag_matching_papers) == 0: # no matching paper from MAG
                    no_matching_paper_count += 1
                    continue
                if len(mag_matching_papers) > 1: # more than one matching papers --> take max cc number
                    citation_count = max([cc["CitationCount"] for cc in mag_matching_papers])
                else:
                    citation_count = mag_matching_papers[0]["CitationCount"]
                citation_numbers.append(1+citation_count)
#     print(venue, citation_numbers)
    paper_data[venue]["NoMatch"] = no_matching_paper_count
    paper_data[venue]["p_NoMatch(%)"] = float(no_matching_paper_count)/float(paper_data[venue]["Total"])*100.0
    paper_data[venue]["GeoMean"] = gmean(citation_numbers)-1
    paper_data[venue]["AriMean"] = np.mean(citation_numbers)-1
with open(venue_category_filename, "r") as fh:
    reader = csv.reader(fh, delimiter=",")
    # skip header row
    next(reader)
    for row in reader:
        venue_name = row[0]
        compute_venue_weight(venue_name)
# read previous weight
with open(previous_weight_filename, "r") as fh:
    reader = csv.reader(fh, delimiter=",")
    # skip header row
    next(reader)
    for row in reader:
        venue_name = row[0]
        pre_amean = row[1]
        pre_gmean = row[2]
        paper_data[venue_name]["old_GeoMean"] = float(pre_gmean)
        paper_data[venue_name]["old_AriMean"] = float(pre_amean)
df = pd.DataFrame(data=paper_data).T
# df.columns.tolist()
def cell_color(val):
    color = 'black'
    if val == 0:
        color = 'red'
    return 'color: %s' % color

df = df.astype({y:int for y in yearranges})
df = df.astype({"Total":int, "NoMatch":int, "AriMean":float, "GeoMean":float, "p_NoMatch(%)":float})
pd.set_option('display.max_rows', len(df))
print("NoMatch: No matching paper from MAG")
print("p_NoMatch(%): 100*NoMatch/Total")
coverage = 100*sum([pv["NoMatch"] for pv in paper_data.values()])/sum([pv["Total"] for pv in paper_data.values()])
print("{}% of the papers did not match from title search".format(coverage))
df.style.applymap(cell_color)
df.to_csv(venue_weight_filename, columns=["AriMean", "GeoMean"])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11966759002770083, 'fraction_numerical': 0.007479224376731302, 'mean_word_length': 3.5593434343434343, 'pattern_counts': {'":': 5, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '244128', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 1228, 'n_tokens_neox': 1139, 'n_words': 280, 'path': 'data/score_venues.ipynb', 'repo_name': 'csmetrics/csmetrics.org', 'size': '3610'}