<jupyter_start><jupyter_text>Descriptive statistics on classes and properties (ITO v0.92)<jupyter_code>from owlready2 import *
from matplotlib.ticker import ScalarFormatter
import rdflib
import pandas as pd
%matplotlib inline
# Paths
ONTOLOGY = get_ontology('ITO-23-09-2021.owl')
#onto = get_ontology("https://github.com/OpenBioLink/ITO/raw/master/ITO.owl")
ONTOLOGY.load()
ITO = ONTOLOGY.get_namespace('https://ai-strategies.org/ontology')
EDAM = ONTOLOGY.get_namespace('http://edamontology.org')
graph = default_world.as_rdflib_graph()

namespaces ={ 'owl': rdflib.OWL,
              'rdfs': rdflib.RDFS,
              'edam': 'http://edamontology.org/',
              'ito': 'https://ai-strategies.org/ontology/'}<jupyter_output><empty_output><jupyter_text>Total number of papers covered by ITO<jupyter_code>result = graph.query("""
SELECT (COUNT(?paper) as ?paper_count) WHERE {
    ?paper a edam:data_0971 .
}
""", initNs = namespaces)
df = pd.DataFrame.from_dict(result, orient='columns')
df.columns = [i for i in result.bindings[0]]
df<jupyter_output><empty_output><jupyter_text>Time span of publications covered by PWC<jupyter_code>result = graph.query("""
SELECT ?paper ?date WHERE {
    ?paper a edam:data_0971 .
    ?paper <http://www.geneontology.org/formats/oboInOwl#date> ?date
}
ORDER BY ?date
""", initNs = namespaces)
df = pd.DataFrame.from_dict(result, orient='columns')
df.columns = [i for i in result.bindings[0]]
df.to_csv('data/time_span_of_publications_covered_by_PWC.csv', index=False)
df

# Plot bar chart

time_span_of_publications_covered_by_PWC = pd.read_csv('data/time_span_of_publications_covered_by_PWC.csv')

# Convert 'date' column to datetime
time_span_of_publications_covered_by_PWC['date'] = pd.to_datetime(time_span_of_publications_covered_by_PWC['date'])

# Add extra column for year
time_span_of_publications_covered_by_PWC['year'] = time_span_of_publications_covered_by_PWC['date'].apply(lambda x: "%d" % (x.year))
time_span_of_publications_covered_by_PWC_grouped_by_year = time_span_of_publications_covered_by_PWC.groupby('year').size()


ax = time_span_of_publications_covered_by_PWC_grouped_by_year.plot.bar(rot=60, color=['#607996'], 
                                                                       figsize=(7,4), log=True)

ax.set_yscale('log')
ax.yaxis.set_major_formatter(ScalarFormatter())
ax.xaxis.set_label_text('Year')
ax.yaxis.set_label_text('Number of publications')

# Save figure
ax.figure.savefig('plots/time_span_of_publications.png', format='png')<jupyter_output><empty_output><jupyter_text>Total number of distinct benchmarks (datasets)<jupyter_code>result = graph.query("""
SELECT (COUNT(?benchmark) as ?benchmark_count) WHERE {
    ?benchmark a <https://identifiers.org/ito:Benchmark_dataset> .
}
""", initNs = namespaces)
df = df.from_dict(result)
df<jupyter_output><empty_output><jupyter_text>Total number of benchmark results<jupyter_code>result = graph.query("""
SELECT (COUNT(?benchmark) as ?benchmark_count) 
WHERE {
    ?benchmark rdf:type/rdfs:subClassOf* <https://identifiers.org/ito:Benchmarking> .
} """, initNs = namespaces)

df = pd.DataFrame.from_dict(result, orient='columns')
df.columns = [i for i in result.bindings[0]]
df<jupyter_output><empty_output><jupyter_text>Number of benchmarks per high-level process<jupyter_code>result = graph.query("""
SELECT ?high_level_process_label ?high_level_process
       (COUNT(?benchmark) AS ?benchmark_count)
WHERE {
    ?high_level_process rdfs:subClassOf <https://identifiers.org/ito:ITO_01625>;
                        rdfs:label ?high_level_process_label . 
    ?benchmark rdfs:subClassOf <https://identifiers.org/ito:Benchmarking> . 

    ?benchmark rdfs:subClassOf* ?high_level_process .
}
GROUP BY ?high_level_process
ORDER BY DESC(?benchmark_count)
""", initNs = namespaces)
number_of_benchmarks = pd.DataFrame.from_dict(result, orient='columns')
number_of_benchmarks.columns = [str(i) for i in result.bindings[0]]
number_of_benchmarks 
#number_of_benchmarks = pd.concat([number_of_benchmarks.iloc[:,-1].astype(float), number_of_benchmarks.iloc[:,0]], axis=1)
#number_of_benchmarks = number_of_benchmarks.drop(index=[0,12])
#number_of_benchmarks<jupyter_output><empty_output><jupyter_text>Number of benchmark results per high level process<jupyter_code>result = graph.query("""
SELECT ?high_level_process_label ?high_level_process
       (COUNT(?benchmark) AS ?benchmark_count)
WHERE {
    ?high_level_process rdfs:subClassOf <https://identifiers.org/ito:ITO_01625>;
                        rdfs:label ?high_level_process_label . 
    ?benchmark rdf:type/rdfs:subClassOf* <https://identifiers.org/ito:Benchmarking> . 
    ?benchmark rdf:type/rdfs:subClassOf* ?high_level_process .
}
GROUP BY ?high_level_process
ORDER BY DESC(?benchmark_count)
""", initNs = namespaces)
number_of_benchmark_results = pd.DataFrame.from_dict(result, orient='columns')
number_of_benchmark_results.columns = [str(i) for i in result.bindings[0]]
#Fix column names
number_of_benchmarks.columns = ['high_level_process_label', 'high_level_process', 'Benchmarks']
number_of_benchmark_results.columns = ['high_level_process_label', 'high_level_process', 'Benchmark results']

number_of_benchmark_results.columns

comparison_df = pd.merge(number_of_benchmarks, number_of_benchmark_results, on=['high_level_process_label', 'high_level_process'])
comparison_df.drop(comparison_df.columns[1], axis=1, inplace=True)
comparison_df = comparison_df.sort_values(by='Benchmarks')
#Drop 'benchmarking' process
comparison_df = comparison_df[:-1]
# Covert to float for plotting
comparison_df['Benchmarks'] = comparison_df['Benchmarks'].astype(float)
comparison_df['Benchmark results'] = comparison_df['Benchmark results'].astype(float)
comparison_df = comparison_df.set_index(comparison_df.iloc[:,0])
ax= comparison_df.plot.barh(figsize=(9,9), color=['#a1cbfb', '#607996'], log=True)


ax.set_xscale('log')
ax.xaxis.set_major_formatter(ScalarFormatter())
#ax.xaxis.set_label_text('Year')
ax.yaxis.set_label_text('Process')
# Save figure
ax.figure.savefig('plots/comparison_benchmarks_and_results.png', format='png', bbox_inches='tight')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1294251380318285, 'fraction_numerical': 0.01039298473530367, 'mean_word_length': 6.501827040194884, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 9, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '190245', 'license': 'MIT', 'n_tokens_mistral': 2244, 'n_tokens_neox': 2158, 'n_words': 334, 'path': 'notebooks/descriptive_statistics_v0.92/statistics_on_classes_and_properties.ipynb', 'repo_name': 'OpenBioLink/ITM', 'size': '6158'}