<jupyter_start><jupyter_text>2: externally inconsistent references<jupyter_code>with open('../2_ext_ref_consistency/results/review_results.json') as fp:
    ex2_results = json.load(fp)

with open('../2_ext_ref_consistency/results/final_suspects.json') as fp:
    ex2_suspects = {
        v['dbpedia']: v
        for v in json.load(fp)
    }

for dbpedia, v in ex2_results.items():
    suspect = ex2_suspects[dbpedia]
    o = {
        'experiment': 'ex2',
        'comment': v['comment'],
        'comment2': suspect['reason'],
    }

    if v['decision'] == 'invalid':
        o['issue'] = 'invalid DBpedia reference'
        for uri in suspect['cso_topics']:
            uri_issues[uri].append(o)

    elif v['decision'] == 'other':
        o['issue'] = 'other'
        for uri in suspect['cso_topics']:
            uri_warnings[uri].append(o)<jupyter_output><empty_output><jupyter_text>3: missing references to corresponding KBs<jupyter_code>ex3_results = pd.read_csv('../3_missing_refs/results/missing_refs.tsv', sep='\t')

for uri in ex3_results['t'].values:
    uri_issues[uri].append({
        'experiment': 'ex3',
        'issue': 'missing Wikidata reference',
    })<jupyter_output><empty_output><jupyter_text>4: logically invalid alignments<jupyter_code>ex4_results = pd.read_csv('../4_refs_reasoning/results/unique_pairs.tsv', sep='\t', names=['a', 'b'])

for _, r in ex4_results.iterrows():
    uri_issues[r.a].append({
        'experiment': 'ex4',
        'issue': 'invalid inferred sameAs',
        'comment': r.b
    })
    uri_issues[r.b].append({
        'experiment': 'ex4',
        'issue': 'invalid inferred sameAs',
        'comment': r.a
    })<jupyter_output><empty_output><jupyter_text>5: intra-cluster alignment inconsistencies<jupyter_code>with open('../5_intra_ref_consistency/results/more_than_one_reference.json') as fp:
    ex5_too_many = json.load(fp)

for v in ex5_too_many:
    cluster_issues[uri_to_cluster[v['topics'][0]]].append({
        'experiment': 'ex5',
        'issue': 'cluster contains more than one unique DBpedia reference',
    })
with open('../5_intra_ref_consistency/results/missing_references.json') as fp:
    ex5_missing = json.load(fp)

for v in ex5_missing:
    for t in v['refs_missing']:
        uri_issues[t].append({
            'experiment': 'ex5',
            'issue': 'missing DBpedia reference',
            'comment': v['dbpedia'],
        })<jupyter_output><empty_output><jupyter_text>6: term conflation<jupyter_code>ex6_results = pd.read_csv('../6_conflation/results/cluster_reviews.csv')

# Assume majority vote approach
for _, r in ex6_results[ex6_results.is_wrong_majority].iterrows():
    cluster_issues[r.cluster].append({
        'experiment': 'ex6',
        'issue': 'inconsistent synonyms in cluster',
    })<jupyter_output><empty_output><jupyter_text>"Propagate" issues from individual URIs to clusters<jupyter_code>for uri, issues in uri_issues.items():
    for issue in issues:
        c_issue = issue.copy()
        c_issue['uri'] = uri
        cluster_issues[uri_to_cluster[uri]].append(c_issue)<jupyter_output><empty_output><jupyter_text>Summarize<jupyter_code>cluster_issue_count = np.array([len(issues) for issues in cluster_issues])

cl_unique, cl_counts = np.unique(cluster_issue_count, return_counts=True)
print(f'Nonzero: {cl_counts[1:].sum()}')
print(np.asarray((cl_unique, cl_counts)).T)
uri_issue_count = np.array([len(issues) for issues in uri_issues.values()])

uri_unique, uri_counts = np.unique(uri_issue_count, return_counts=True)
print(f'Nonzero: {uri_counts[1:].sum()}')
print(np.asarray((uri_unique, uri_counts)).T)<jupyter_output>Nonzero: 4287
[[    0 10003]
 [    1  1875]
 [    2   610]
 [    3   439]
 [    4   247]
 [    5   224]
 [    6   129]
 [    7   144]
 [    8   147]
 [    9    91]
 [   10    79]
 [   11    73]
 [   12    46]
 [   13    24]
 [   14    19]
 [   15     7]
 [   16    10]
 [   17    12]
 [   18    12]
 [   19    44]
 [   24    25]
 [   27     3]
 [   28     6]
 [   29    21]]
{'fraction_non_alphanumeric': 0.14717641179410296, 'fraction_numerical': 0.03423288355822089, 'mean_word_length': 3.374863387978142, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '249784', 'license': 'MIT', 'n_tokens_mistral': 1534, 'n_tokens_neox': 1456, 'n_words': 268, 'path': 'cso/summary/summary.ipynb', 'repo_name': 'Ostrzyciel/ontology-quality-2022', 'size': '4002'}