<jupyter_start><jupyter_text>Fair Scraper<jupyter_code>"""
Scrapes information from FAIRsharing.org
"""
import os
import requests
from lxml import html
import pandas as pd
import json
#from ncats_translator_dqa import config
%load_ext watermark
%watermark -a 'Maastricht University' -nmv --packages requests,lxml,pandas,json
import sys
sys.path.insert(0, '/Users/pedrohserrano/NCATS-Translator-DQA') 
import ncats_translator_dqa as dqa<jupyter_output><empty_output><jupyter_text>fair_scraper(url)FAIRsharing.org for some basic information.    Scrapes FAIRsharing.org for some basic information, including title, scope and data types, terminology artifacts,    and conditions of use.    :param url: String url to page to scrape    :return: FAIRPrelimStats object - Which url<jupyter_code>url = 'https://fairsharing.org/biodbcore-000015'
# output message
#if config.verbose:
#    print('Scraping: ' + url)
# load the page
page = requests.get(url)
# parse the HTML
html_content = html.fromstring(page.content)<jupyter_output><empty_output><jupyter_text>- Get the title<jupyter_code>title = html_content.xpath('//div[@class="title-text"]/h2/text()[last()]')
title = title[0].strip()
title = [title]
title<jupyter_output><empty_output><jupyter_text>- Get the tags<jupyter_code># Find the listed items under Scope and data types
# <li class="bio-tag domain">
#     <span class="bio-icon-tag"style="padding-right: 5px"></span>
#     Approved drug
# </li>
sad = html_content.xpath('//li[@class="bio-tag domain"]/text()[last()]')
sad = [x.strip() for x in sad]
sad<jupyter_output><empty_output><jupyter_text>- Get the terminology artifacts<jupyter_code># Find the list items under Terminology Artifacts
# <p><span class="heavier">Terminology Artifacts</span></p>
# <ul class="record-list-link">
# 	<li class="small"><a href="/bsg-s000039" target="_blank">Chemical Entities of Biological Interest</a></li>
# 	<li class="small"><a href="/bsg-s000136" target="_blank">PSI Molecular Interaction Controlled Vocabulary</a></li>
# </ul>
ta = html_content.xpath('//span[text()="Terminology Artifacts"]/../../ul/li/a/text()')
ta = [x.strip() for x in ta]
ta<jupyter_output><empty_output><jupyter_text>- Get the license<jupyter_code># Get license
# <div class="standard-unit">
    # <p class="section-title"><span class="heavier">Conditions of Use</span></p>
lic_groups = html_content.xpath('//span[text()="Conditions of Use"]/../../span[@class="section-header"]')

lic_info = []
for lic_group in lic_groups:
    applies_to = lic_group.xpath('text()') # Get the "Applies to" text and fix weird whitespace
    applies_to = ' '.join(applies_to[0].split())
    licenses = lic_group.xpath('following-sibling::ul[1]/li/span//text()')     # Get the licenses
    licenses = [x.strip() for x in licenses]
    lic_info.append((applies_to, licenses))     # Add the license information as a tuple
lic_info
lic_strings = []
sep = '; '
for lic in lic_info:
    lic_strings.append(lic[0] + " = {" + sep.join(lic[1]) + "}")
    lic_string = sep.join(lic_strings)
licence = [lic_string]
licence<jupyter_output><empty_output><jupyter_text>- FAIR Scrapper elements  url, title, sad, ta, lic_info  fair_table(fpss, file_output)Writes a list of preliminary statistics from multiple FAIRsharing.org urls to a CSV file    :param fpss: List of FAIRPrelimStats    :param file_output: Path to output file to write to (String)    :return:<jupyter_code>fpss = [[url], title, sad, ta, licence]
fpss
num_fpss = len(fpss)
num_fpss
titles = ['url', 'title', 'coverage', 'vocabReuse', 'license']
summary = {key: value for (key, value) in zip(titles, fpss)}
summary
def writeToJSONFile(path, fileName, data):
    filePathNameWExt = './' + path + '/' + fileName + '.json'
    with open(filePathNameWExt, 'w') as fp:
        json.dump(data, fp)
writeToJSONFile('./','metrics',summary)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1489637305699482, 'fraction_numerical': 0.006735751295336788, 'mean_word_length': 5.467336683417085, 'pattern_counts': {'":': 0, '<': 52, '<?xml version=': 0, '>': 52, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '447488', 'license': 'MIT', 'n_tokens_mistral': 1382, 'n_tokens_neox': 1342, 'n_words': 324, 'path': 'notebooks/scrapper.ipynb', 'repo_name': 'amalic/fairsharing-metrics', 'size': '3860'}