<jupyter_start><jupyter_text>Process RoMEO PoliciesUses some methods and code from [`01.romeo.ipynb`](https://github.com/asapbio/publisher-agreements/blob/8adacf6b1020aa82ab7b5d4c9e02af516e5f3779/01.romeo.ipynb) in `asapbio/publisher-agreements`.<jupyter_code>import json
import os
import pathlib
from urllib.request import urlretrieve

import lxml.etree
import pandas
import networkx<jupyter_output><empty_output><jupyter_text>RoMEO DownloadsSee http://www.sherpa.ac.uk/downloads/. Files are not downloaded if they exist. Hence to redownload RoMEO data, delete the files in `downloads`.<jupyter_code>def get_romeo_api_key():
    """
    The RoMEO API key can either be set using secrets.py or via the
    ROMEO_API_KEY environment variable.
    """
    try:
        from secrets import romeo_api_key
    except ImportError:
        romeo_api_key = os.environ['ROMEO_API_KEY']
    return romeo_api_key
files = [
    ('journal-title-issns', 'tsv'),
    ('journal-titles', 'tsv'),
    ('policies-summary', 'tsv'),
    ('policies-summary', 'xml'),
]
for name, extension in files:
    path = pathlib.Path(f'downloads/{name}.{extension}')
    if path.is_file():
        continue
    romeo_api_key = get_romeo_api_key()
    url = f'http://www.sherpa.ac.uk/downloads/{name}/{romeo_api_key}/{extension}/'
    path, headers = urlretrieve(url, path)
    timestamp = headers.get('Date')
    print(f'Downloaded {path} on {timestamp}')
path = pathlib.Path('downloads/policies-full.xml')
if not path.is_file():
    romeo_api_key = get_romeo_api_key()
    url = f'http://www.sherpa.ac.uk/romeo/api29.php?all=yes&ak={romeo_api_key}'
    path, headers = urlretrieve(url, path)
    headers.get('Date')<jupyter_output><empty_output><jupyter_text>Construct ontology<jupyter_code>graph = networkx.DiGraph(
    name='SHERPA RoMEO',
    license='CC BY-NC-SA 2.5',
    homepage='http://www.sherpa.ac.uk/romeo/index.php',
)
policy_summary_df = pandas.read_table('downloads/policies-summary.tsv')
policy_summary_df = policy_summary_df.sort_values('RoMEO Record ID')
policy_summary_df.head(2)
def format_romeo_id(number):
    if number is None or pandas.isna(number):
        return None
    return f'romeo_{number}'
# Create policy nodes
policies = json.loads(policy_summary_df.to_json(orient='records'))
for policy in policies:
    record_id = format_romeo_id(policy['RoMEO Record ID'])
    graph.add_node(record_id, **policy)

len(graph)
# Create policy (parent) relationships
tree = lxml.etree.parse('downloads/policies-full.xml')
policies = tree.findall('publishers/publisher')
policies.sort(key=lambda x: int(x.get('id')))
for policy in policies:
    policy_id = format_romeo_id(policy.get('id'))
    parent_id = format_romeo_id(policy.get('parentid'))
    if not parent_id:
        continue
    graph.add_edge(policy_id, parent_id)

assert networkx.is_directed_acyclic_graph(graph)
graph.number_of_edges()
# # Print a single policy
# policy = policies[0]
# print(lxml.etree.tostring(policy, encoding=str))
# for child, parent in graph.edges():
#     child_data = graph.node[child]
#     parent_data = graph.node[parent]
# Draw the policy graph (without journal nodes)
%matplotlib inline
connected_nodes = [node for node, degree in graph.degree(graph) if degree > 0]
len(connected_nodes)
connected = graph.subgraph(connected_nodes)
# https://networkx.github.io/documentation/stable/reference/generated/networkx.drawing.nx_pylab.draw_networkx.html#networkx.drawing.nx_pylab.draw_networkx
networkx.draw(connected, alpha=0.5, node_size=100, node_color='#1c64d8')
with open('downloads/journal-title-issns.tsv', encoding='utf-8', errors='ignore') as read_file:
    journal_df = pandas.read_table(read_file)
journal_df = journal_df.sort_values(['RoMEO Record ID', 'Journal Title'])
journal_df['romeo_id'] = journal_df['RoMEO Record ID'].map(format_romeo_id)
journal_df.head(4)
id_to_journals = dict(
    journal_df.groupby('romeo_id', sort=False)
    ['Journal Title'].unique().map(list)
)
json_str = json.dumps(id_to_journals, indent=2, ensure_ascii=False)
pathlib.Path('data/romeo_id-to-journals.json').write_text(json_str)
# Create journal to policy relationships
# use to_json rather than to_dict to convert NaN to None
journals = json.loads(journal_df.to_json(orient='records'))
for journal in journals:
    journal_title = journal['Journal Title']
    parent = journal.pop('romeo_id')
    graph.add_node(journal_title, **journal)
    graph.add_edge(journal_title, parent)
graph.number_of_nodes(), graph.number_of_edges()
# Export as node link data JSON
# https://networkx.github.io/documentation/stable/reference/readwrite/generated/networkx.readwrite.json_graph.node_link_data.html#networkx.readwrite.json_graph.node_link_data
node_link_data = networkx.node_link_data(graph)
with open('data/ontology.json', 'w') as write_file:
    json.dump(node_link_data, write_file, indent=2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12786415711947627, 'fraction_numerical': 0.009410801963993453, 'mean_word_length': 5.449868073878628, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 3, 'lorem ipsum': 0, 'www.': 4, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '590665', 'license': 'CC0-1.0', 'n_tokens_mistral': 1747, 'n_tokens_neox': 1720, 'n_words': 314, 'path': 'romeo/01.romeo.ipynb', 'repo_name': 'transpose-publishing/policies-database', 'size': '4888'}