<jupyter_start><jupyter_text>Get 202 clinical trials of Brest Cancer<jupyter_code>hundred_url = 'https://clinicaltrials.gov/api/query/study_fields?expr=breast+cancer&fields=NCTId&min_rnk=1&max_rnk=200&fmt=json'
ID = requests.get(hundred_url).json()['StudyFieldsResponse']['StudyFields']
IDs = []
for id in ID:
    IDs.append(id['NCTId'][0])

len(a)
count = 0
for i in a:
    output = {}
    generate_Graph(i,crawling_data(i,output))
    print(i,count+1)
    count += 1

for i in range(195,len(IDs)):
    print(IDs[i],i+1)
    output = {}
    generate_Graph(IDs[i],crawling_data(IDs[i],output))
    

kg = Graph()
kg.bind('Clinical_Trials', CT)
kg.bind('schema',SCHEMA)
ct_uri = URIRef('https://clinicaltrials.gov/ct2/show/'+ID)
kg.add((ct_uri, RDF.type, SCHEMA.MedicalTrial))
kg.add((ct_uri, CT.hasID, Literal(ID)))
for i in output:
    if (len(out["StudyType"]) > 0):
        kg.add((ct_uri, CT.hasStudyType, Literal(output['StudyType'])))
    if (i == 'InclusionCriteria' and len(output['InclusionCriteria'])>0):
        for j in output[i]:
            kg.add((ct_uri, CT.hasInclusionCriteria, Literal(j)))
    if (i == 'ExclusionCriteria' and len(output['ExclusionCriteria'])>0):
        for j in output[i]:
            kg.add((ct_uri, CT.hasExclusionCriteria, Literal(j)))
    if (i == 'Drugs' and len(output['Drugs'])>0):
        for j in output[i]:
            k = URIRef(output[i][j]['url'])
            kg.add((ct_uri, CT.hasDrug, k))
            kg.add((k, SCHEMA.name, Literal(j)))
            kg.add((k, SCHEMA.description, Literal(output[i][j]['description'])))
    
    l = len(output["groups"])

    out_field = ID+'_outcome'
    kg.add((ct_uri, CT.hasOutcome,CT[out_field]))
    
    treatment_field = ID+'_treamentDescription'
    kg.add((ct_uri, CT.hasTreatmentPlan,CT[treatment_field]))

    nofP_field = ID+'_numberofPatients'
    kg.add((ct_uri, CT.hasNumberofPatients,CT[nofP_field]))
    
    aofP_field = 'NCT04199078_ageOfPatients'
    kg.add((ct_uri, CT.hasAge,CT[aofP_field]))
    
    if l > 0:
        for i in range(0,len(outcome["groups"])):
            g = outcome["groups"][i].replace(' ',"_") + "_Group"
            g0 = g + "_outcome"
            kg.add((CT[out_field], CT.hasGroup, CT(g0)))
            kg.add((CT[g0], CT.hasOutcomeDescription, Literal(output["outcome_description"])))
            if (len(output["outcome_measurements"]) == 2):
                kg.add((CT[g0], CT.hasMeasurement, Literal(output["outcome_measurements"][0])))
                kg.add((CT[g0], CT.hasUnit, Literal(output["outcome_measurements"][1])))
            kg.add((CT(g0),SCHEMA.number,Literal(output['outcome'][i])))

            g1 = g + '_treatment'
            kg.add((CT[treatment_field], CT.hasTreatment,CT[g1]))
            if (len(output["treatment"]) == l):
                kg.add((CT[g1], CT.hasGroup, Literal(g)))
                kg.add((CT[g1], CT.hasdescription, Literal(output["treatment"][i])))

            g2 = g + '_numberOfPatients'
            kg.add((CT[nofP_field], CT.hasTreatment,CT[g2]))
            if (len(output["numberOfPatients"]) == l):
                kg.add((CT[g2], CT.hasGroup, Literal(g)))
                kg.add((CT[g2], SCHEMA.number, Literal(output["numberOfPatients"][i])))
                kg.add((CT[g2], CT.hasUnit, Literal("person")))
    
            g3 = g + '_ageOfPatients'
            kg.add((CT[aofP_field], CT.hasAgeofPatients,CT[g3]))
            if (len(output["age_measurements"]) == l):
                kg.add((CT[g3], CT.hasGroup, Literal(g)))
                kg.add((CT[g3], SCHEMA.number, Literal(output["ageOfPatients"][i])))
            if (len(output["age_measurements"][0]) == 2):
                kg.add((CT[g3], CT.hasType, Literal(output["age_measurements"][0])))
                kg.add((CT[g3], CT.hasUnit, Literal(output["age_measurements"][1])))


        hazardRatio = ID + '_hazardRatio'
        kg.add((ct_uri, CT.hasHazardRatio, CT[hazardRatio]))
        kg.add((CT[hazardRatio], SCHEMA.number, Literal(output["hazardRatio"])))

kg.serialize('100_graphs/'+ID'.ttl', format="turtle")    
url = 'https://clinicaltrials.gov/ct2/show/results/NCT01740427?view=results'
    
response = urlopen(url)
soup = BS(response, 'html.parser')
baseline = soup.find_all("th",class_="de-popFlowLabelCell")
response = urlopen(url)
soup = BS(response, 'html.parser')
dom = etree.HTML(str(soup))

hr = str(dom.xpath('//*[@id="EXPAND-stat-analysis-1-1"]/table/tbody/tr[12]/td/text()')[0]).strip('\n').strip()
hr
NCT03093350 = dict()
studytype_url = 'https://clinicaltrials.gov/api/query/study_fields?expr=NCT03093350&fields=studytype&min_rnk=1&max_rnk=&fmt=json'
studytype = requests.get(studytype_url).json()['StudyFieldsResponse']['StudyFields'][0]['StudyType'][0]
NCT03093350['StudyType'] = studytype
criteria_url = 'https://clinicaltrials.gov/api/query/full_studies?expr=NCT04176354&min_rnk=1&max_rnk=&fmt=json'
criteria = requests.get(criteria_url).json()['FullStudiesResponse']['FullStudies'][0]['Study']['ProtocolSection']['EligibilityModule']['EligibilityCriteria']

b = criteria.split('\n')
inclusion = []
exclusion = []
for i in range(0,len(b)):
    if b[i] == 'Inclusion Criteria:':
        for j in range(i+1,len(b)):
            if b[j] == 'Exclusion Criteria:':
                break
            if b[j] != '':
                inclusion.append(b[j])
    if b[i] == 'Exclusion Criteria:':
        for j in range(i+1,len(b)):
            if b[j] != '':
                exclusion.append(b[j])
NCT03093350['InclusionCriteria'] = inclusion
NCT03093350['ExclusionCriteria'] = exclusion

drug_url = 'https://clinicaltrials.gov/api/query/study_fields?expr=NCT04176354&fields=ArmGroupInterventionName&min_rnk=1&max_rnk=&fmt=json'
drug = requests.get(drug_url).json()['StudyFieldsResponse']['StudyFields'][0]['ArmGroupInterventionName']
drug
drugs = [i.strip("Drug:").strip("'").strip() for i in drug] 
drugs2 = set()
for i in drugs:
    for j in i.split(" "):
        drugs2.add(j)
# drugs = list(dict.fromkeys(drugs))
drugs2.remove('an')
drugs2.remove('+')
drugs2.remove('aromatase')
drugs2.remove('inhibito')

API_ENDPOINT = 'https://en.wikipedia.org/w/api.php'
drugs_with_des = {}
for i in drugs2:
    drugs_with_des[i] = {}
    query = i
    params = {
        'action':'query',
        'list':'search',
        'srsearch':query,
        'format':'json',
    }
    drugs_with_des[i]['url'] = requests.get(API_ENDPOINT,params=params).url
    drugs_with_des[i]['description'] = requests.get(API_ENDPOINT,params=params).json()\
    ['query']['search'][0]['snippet']
drugs_with_des
NCT03093350['Drugs'] = drugs_with_des



url = "https://clinicaltrials.gov/ct2/show/results/NCT03093350?view=results"
response = urlopen(url)
soup = BS(response, 'html.parser')
baseline = soup.find_all("th",class_="de-popFlowLabelCell")
groups = []
number_of_patients = []
for i in range(0,len(baseline)):
    groups.append(baseline[i].text.strip("\n").strip("\r").strip())
    number_of_patients.append(soup.find_all("td",class_="de-numValue_baselineDataCell")[i].text.strip("\n").strip("\r").strip())

response = urlopen(url)
soup = BS(response, 'html.parser')
dom = etree.HTML(str(soup))


monaleesa2["numberOfPatients"] = []
monaleesa2["numberOfPatients"].append(soup.find_all("td", class_="de-numValue_popFlowCell")[0].text.strip("\r").strip("\n").strip())
monaleesa2["numberOfPatients"].append(soup.find_all("td", class_="de-numValue_popFlowCell")[1].text.strip("\r").strip("\n").strip())
monaleesa2["outcome"] = []
monaleesa2["outcome"].append(soup.find_all("td", class_="de-numValue_outcomeDataCell")[2].text.strip('\n').strip().split(" ")[0])
monaleesa2["outcome"].append(soup.find_all("td", class_="de-numValue_outcomeDataCell")[3].text.strip('\n').strip().split(" ")[0])
monaleesa2["treatment"] = []
monaleesa2["treatment"].append(soup.find_all("td",class_="de-outcomeLabelCell")[7].text.strip('\n').strip('\r').strip())
monaleesa2["treatment"].append(soup.find_all("td",class_="de-outcomeLabelCell")[8].text.strip('\n').strip('\r').strip())
monaleesa2["ageOfPatients"] = []
monaleesa2["ageOfPatients"].append(soup.find_all("td",class_="de-numValue_baselineDataCell")[11].text.strip('\n').strip('\r').strip().split(" ")[0])
monaleesa2["ageOfPatients"].append(soup.find_all("td",class_="de-numValue_baselineDataCell")[12].text.strip('\n').strip('\r').strip().split(" ")[0])
monaleesa2["hazardRatio"] = soup.find_all("td",class_="de-outcomeDataCell")[15].text.strip('\n').strip('\r').strip()

monaleesa2['PFS'] = [dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[6]/td[1]/div[1]/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''),
                  dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[6]/td[2]/div[1]/text()')[0].strip('\n').strip().strip('\n').strip()]
monaleesa2["outcome_measurements"] = [dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[5]/th/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''),
                             dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[5]/th/div/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')]
monaleesa2["age_measurements"] = [dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[7]/th[1]/div/div[1]/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''),
                             dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[7]/th[1]/div/div[2]/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')]





NCT03093350["numberOfPatients"] = str(dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[4]/td/text()')[0]).strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')
    
NCT03093350["groupTitle"] = str(dom.xpath('//*[@id="tab-body"]/div[1]/div[2]/div[2]/table[2]/tbody/tr[1]/th/text()')[0]).strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')

NCT03093350["groupDescription"] = str(dom.xpath('//*[@id="EXPAND-armGroupDescriptionRow-baseline"]/td[2]//p/text()'))

NCT03093350["ageOfPatients"] = str(dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[9]/td[2]/span/text()')[0]).strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')

NCT03093350["PFS"] = str(dom.xpath('//*[@id="EXPAND-outcome-data-2"]/table/tbody/tr[6]/td/div[1]/text()')[0]).strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')    

NCT03093350["PFS_unit"] = str(dom.xpath('//*[@id="EXPAND-outcome-data-2"]/table/tbody/tr[5]/th/div/text()')[0]).strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','') 

NCT03093350["OAS"] = str(dom.xpath('//*[@id="EXPAND-outcome-data-3"]/table/tbody/tr[6]/td/div[1]/text()')[0]).strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')    

NCT03093350["OAS_unit"] = str(dom.xpath('//*[@id="EXPAND-outcome-data-3"]/table/tbody/tr[5]/th/div/text()')[0]).strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','') 

kg_NCT03093350 = Graph()
kg_NCT03093350.bind('Clinical_Trials', CT)
kg_NCT03093350.bind('schema',SCHEMA)
kg_NCT03093350.add((ct_uri, CT.hasID, Literal('NCT03093350')))
            
ct_uri = URIRef('https://clinicaltrials.gov/ct2/show/NCT03093350')
kg_NCT03093350.add((ct_uri, RDF.type, SCHEMA.MedicalTrial))
title = str(NCT03093350["groupTitle"].replace(' ','_'))
kg_NCT03093350.add((ct_uri, CT.hasGroup, CT[title]))
kg_NCT03093350.add((CT[title], CT.hasDescription, Literal(NCT03093350["groupDescription"])))

kg_NCT03093350.add((ct_uri, CT.hasNumberofPatients, Literal(NCT03093350["numberOfPatients"])))
kg_NCT03093350.add((ct_uri, CT.hasMedianAge, Literal(NCT03093350["ageOfPatients"])))
kg_NCT03093350.add((ct_uri, CT.hasPFS, CT['NCT03093350_PFS']))
kg_NCT03093350.add((CT['NCT03093350_PFS'], CT.hasUnit, Literal(NCT03093350["PFS_unit"])))
kg_NCT03093350.add((CT['NCT03093350_PFS'], SCHEMA.number, Literal(NCT03093350["PFS"])))
kg_NCT03093350.add((ct_uri, CT.hasOAS, CT['NCT03093350_OAS']))
kg_NCT03093350.add((CT['NCT03093350_OAS'], CT.hasUnit, Literal(NCT03093350["OAS_unit"])))
kg_NCT03093350.add((CT['NCT03093350_OAS'], SCHEMA.number, Literal(NCT03093350["OAS"])))  

for i in NCT03093350:
    kg_NCT03093350.add((ct_uri, CT.hasStudyType, Literal(NCT03093350['StudyType'])))
    if i == 'InclusionCriteria':
        for j in NCT03093350[i]:
            kg_NCT03093350.add((ct_uri, CT.hasInclusionCriteria, Literal(j)))
    if i == 'ExclusionCriteria':
        for j in NCT03093350[i]:
            kg_NCT03093350.add((ct_uri, CT.hasExclusionCriteria, Literal(j)))
    if i == 'Drugs':
        for j in NCT03093350[i]:
            k = URIRef(NCT03093350[i][j]['url'])
            kg_NCT03093350.add((ct_uri, CT.hasDrug, k))
            kg_NCT03093350.add((k, SCHEMA.name, Literal(j)))
            kg_NCT03093350.add((k, SCHEMA.description, Literal(NCT03093350[i][j]['description'])))
            
kg_NCT03093350.serialize('new_graphs/NCT03093350.ttl',format="turtle")<jupyter_output><empty_output><jupyter_text>Extract all critera<jupyter_code>paloma2 = dict()
studytype_url = 'https://clinicaltrials.gov/api/query/field_values?expr=paloma2&field=StudyType&fmt=json'
studytype = requests.get(studytype_url).json()['FieldValuesResponse']['FieldValues'][0]['FieldValue']
paloma2['StudyType'] = studytype
criteria_url = 'https://clinicaltrials.gov/api/query/full_studies?expr=paloma2&min_rnk=1&max_rnk=&fmt=json'
criteria = requests.get(criteria_url).json()['FullStudiesResponse']['FullStudies'][0]['Study']['ProtocolSection']['EligibilityModule']['EligibilityCriteria']
criteria
a = criteria.split('\n')
inclusion = []
exclusion = []
for i in range(0,len(a)):
    if a[i] == 'Inclusion Criteria:':
        for j in range(i+1,len(a)):
            if a[j] == 'Exclusion Criteria:':
                break
            if a[j] != '':
                inclusion.append(a[j])
    if a[i] == 'Exclusion Criteria:':
        for j in range(i+1,len(a)):
            if a[j] != '':
                exclusion.append(a[j])
paloma2['InclusionCriteria'] = inclusion
paloma2['ExclusionCriteria'] = exclusion
paloma2
drug_url = 'https://clinicaltrials.gov/api/query/study_fields?expr=paloma-2&fields=ArmGroupInterventionName&min_rnk=2&max_rnk=2&fmt=json'
drug = requests.get(drug_url).json()['StudyFieldsResponse']['StudyFields'][0]['ArmGroupInterventionName']
drug
drugs = [i.strip("Drug:").strip("'").strip() for i in drug] 
drugs = list(dict.fromkeys(drugs))
drugs
API_ENDPOINT = 'https://www.wikidata.org/w/api.php'
drugs_with_des = {}
for i in drugs:
    query = i
    params = {
        'action':'wbsearchentities',
        'format':'json',
        'language':'en',
        'search':query,
    }
    drugs_with_des[i] = requests.get(API_ENDPOINT,params=params).json()['search'][0]['description']
drugs
API_ENDPOINT = 'https://en.wikipedia.org/w/api.php'
query = 'Placebo'
params = {
    'action':'query',
    'list':'search',
    'srsearch':query,
    'format':'json',
}
a = requests.get(API_ENDPOINT,params=params).json()['query']['search'][0]['snippet']
a
API_ENDPOINT = 'https://en.wikipedia.org/w/api.php'
drugs_with_des = {}
for i in drugs:
    drugs_with_des[i] = {}
    query = i
    params = {
        'action':'query',
        'list':'search',
        'srsearch':query,
        'format':'json',
    }
    drugs_with_des[i]['url'] = requests.get(API_ENDPOINT,params=params).url
    drugs_with_des[i]['description'] = requests.get(API_ENDPOINT,params=params).json()\
    ['query']['search'][0]['snippet']
drugs_with_des
API_ENDPOINT = 'https://en.wikipedia.org/w/api.php'
query = 'Letrozole'
params = {
        'action':'query',
        'list':'search',
        'srsearch':query,
        'format':'json',
    }
res = requests.get(API_ENDPOINT,params=params).url
res
paloma2['Drugs'] = drugs_with_des
paloma2
# from tabula import read_pdf
# table = read_pdf('paloma2.pdf',pages = 4,output_format='json')
def get_vals(test_dict, key): 
    for i, j in test_dict.items(): 
        if i == key: 
            yield (j) 
        yield from [] if not isinstance(j, dict) else get_vals(j, key)

# data = []
# for i in table[0]['data']:
#     for j in i:
#         data.append(list(get_vals(j,'text')))
# data = list(filter(lambda a: a != [''],data))
# paloma2['data'] = data
# outcome_url = "https://clinicaltrials.gov/api/query/study_fields?expr=paloma2&fields=OutcomeMeasurementValue&min_rnk=1&max_rnk=&fmt=json"
# outcome = requests.get(outcome_url).json()["StudyFieldsResponse"]["StudyFields"][1]["OutcomeMeasurementValue"][:2]

# treatment_url = "https://clinicaltrials.gov/api/query/study_fields?expr=paloma2&fields=ArmGroupDescription&min_rnk=1&max_rnk=&fmt=json"
# treatment = requests.get(treatment_url).json()["StudyFieldsResponse"]["StudyFields"][1]["ArmGroupDescription"]

# numberOfPatients_url = "https://clinicaltrials.gov/api/query/study_fields?expr=paloma2&fields=BaselineDenomCountValue&min_rnk=1&max_rnk=&fmt=json"
# numberOfPatients = requests.get(numberOfPatients_url).json()["StudyFieldsResponse"]["StudyFields"][1]["BaselineDenomCountValue"]

# ageOfPatients_url = "https://clinicaltrials.gov/api/query/study_fields?expr=paloma2&fields=BaselineMeasurementValue&min_rnk=1&max_rnk=&fmt=json"
# ageOfPatients = requests.get(ageOfPatients_url).json()["StudyFieldsResponse"]["StudyFields"][1]["BaselineMeasurementValue"][:3]
# ageOfPatients

# hazardRatio_url = "https://clinicaltrials.gov/api/query/study_fields?expr=paloma2&fields=OutcomeAnalysisParamValue&min_rnk=1&max_rnk=&fmt=json"
# hazardRatio = requests.get(hazardRatio_url).json()["StudyFieldsResponse"]["StudyFields"][1]["OutcomeAnalysisParamValue"][0]
# hazardRatio
outcome_url = "https://clinicaltrials.gov/api/query/study_fields?expr=paloma2&fields=OutcomeMeasurementValue&min_rnk=1&max_rnk=&fmt=json"
outcome = requests.get(outcome_url).json()["StudyFieldsResponse"]["StudyFields"][1]["OutcomeMeasurementValue"][:2]

treatment_url = "https://clinicaltrials.gov/api/query/study_fields?expr=paloma2&fields=ArmGroupDescription&min_rnk=1&max_rnk=&fmt=json"
treatment = requests.get(treatment_url).json()["StudyFieldsResponse"]["StudyFields"][1]["ArmGroupDescription"]
treatment
numberOfPatients_url = "https://clinicaltrials.gov/api/query/study_fields?expr=paloma2&fields=BaselineDenomCountValue&min_rnk=1&max_rnk=&fmt=json"
numberOfPatients = requests.get(numberOfPatients_url).json()["StudyFieldsResponse"]["StudyFields"][1]["BaselineDenomCountValue"]
numberOfPatients
ageOfPatients_url = "https://clinicaltrials.gov/api/query/study_fields?expr=paloma2&fields=BaselineMeasurementValue&min_rnk=1&max_rnk=&fmt=json"
ageOfPatients = requests.get(ageOfPatients_url).json()["StudyFieldsResponse"]["StudyFields"][1]["BaselineMeasurementValue"][:3]
ageOfPatients
hazardRatio_url = "https://clinicaltrials.gov/api/query/study_fields?expr=paloma2&fields=OutcomeAnalysisParamValue&min_rnk=1&max_rnk=&fmt=json"
hazardRatio = requests.get(hazardRatio_url).json()["StudyFieldsResponse"]["StudyFields"][1]["OutcomeAnalysisParamValue"][0]
hazardRatio
# paloma2["outcome"] = outcome
# paloma2["treatment"] = treatment
# paloma2["numberOfPatients"] = numberOfPatients
# paloma2["ageOfPatients"] = ageOfPatients
# paloma2["hazardRatio"] = hazardRatio
paloma2["hasPercentageOfWhite"] = [77.5,77.5]
paloma2["hasPercentageOfPatientsWithVisceralDisease"] = [48.2,49.5]
paloma2["hasPercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy"] = [48,49.1]
paloma2["hasPercentageOfPatientsWithNewlyDiagonosedAdavancedDisease"] = 37.2
paloma2
from rdflib import Graph, URIRef, Literal, XSD, Namespace, RDF, RDFS
import pandas as pd
import json
CT = Namespace('http://myclinicaltrials.org/CT#')
SCHEMA = Namespace('http://schema.org/')
# kg = Graph()
# kg.bind('Clinical_Trials', CT)
# kg.bind('schema',SCHEMA)<jupyter_output><empty_output><jupyter_text>crawling outcome from html page<jupyter_code>from bs4 import BeautifulSoup as BS
from urllib.request import urlopen
# url = "https://clinicaltrials.gov/ct2/show/results/NCT01740427?cond=paloma2&draw=2&rank=1&view=results"
# html = urlopen(url)
# soup = BS(html, 'lxml')
# title = soup.title
# print(title)
url = "https://clinicaltrials.gov/ct2/show/results/NCT01740427?view=results"
response = urlopen(url)
soup = BS(response, 'html.parser')
dom = etree.HTML(str(soup))
baseline = soup.find_all("th",class_="de-popFlowLabelCell")
groups = []
number_of_patients = []
for i in range(0,len(baseline)):
    groups.append(baseline[i].text.strip("\n").strip("\r").strip())
    number_of_patients.append(soup.find_all("td",class_="de-numValue_baselineDataCell")[i].text.strip("\n").strip("\r").strip())
paloma2["groups"] = groups
paloma2["numberOfPatients"] = number_of_patients
paloma2["ageOfPatients"] = [dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[9]/td[2]/span/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''),
                               dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[9]/td[3]/span/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')]


paloma2
from lxml import etree


paloma2["ageOfPatients"] = [str(dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[9]/td[2]/span/text()')[0])
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''),
                           str(dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[9]/td[3]/span/text()')[0])
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')]
paloma2["PFS"] = [dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[6]/td[1]/div[1]/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''),
                  dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[6]/td[2]/div[1]/text()')[0].strip('\n').strip().strip('\n').strip()]

paloma2["outcome_measurements"] = [dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[5]/th/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''),
                             dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[5]/th/div/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')]
paloma2["age_measurements"] = [dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[7]/th[1]/div/div[1]/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''),
                             dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[7]/th[1]/div/div[2]/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')]
a = dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[6]/td[2]/div[1]/text()')[0].strip('\n').strip().strip('\n').strip()
b = dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[6]/td[1]/div[1]/text()')[0].strip('\n').strip().strip('\n').strip()
b
paloma2
url = "https://clinicaltrials.gov/ct2/show/results/NCT01740427?view=results"
html = urlopen(url)
soup1 = BS(html, 'html.parser')
treatment_number = soup1.find_all("td", class_="de-numValue_popFlowCell")[0].text.strip("\r").strip("\n").strip()
control_number = soup1.find_all("td", class_="de-numValue_popFlowCell")[1].text.strip("\r").strip("\n").strip()
treatment_pfs = soup1.find_all("td", class_="de-numValue_outcomeDataCell")[2].text.strip('\n').strip().split(" ")[0]
control_pfs = soup1.find_all("td", class_="de-numValue_outcomeDataCell")[3].text.strip('\n').strip().split(" ")[0]
treatment_plan = soup1.find_all("td",class_="de-outcomeLabelCell")[7].text.strip('\n').strip('\r').strip()
control_plan = soup1.find_all("td",class_="de-outcomeLabelCell")[8].text.strip('\n').strip('\r').strip()
treatment_age = soup1.find_all("td",class_="de-numValue_baselineDataCell")[11].text.strip('\n').strip('\r').strip().split(" ")[0]
control_age = soup1.find_all("td",class_="de-numValue_baselineDataCell")[12].text.strip('\n').strip('\r').strip().split(" ")[0]
hazard = soup1.find_all("td",class_="de-outcomeDataCell")[15].text.strip('\n').strip('\r').strip()
paloma2["numberOfPatients"] = [treatment_number,control_number]
paloma2["outcome"] = [treatment_pfs,control_pfs]
paloma2["treatment"] = [treatment_plan,control_plan]
paloma2["ageOfPatients"] = [treatment_age, control_age]
paloma2['hazardRatio'] = hazard
kg = Graph()
kg.bind('Clinical_Trials', CT)
kg.bind('schema',SCHEMA)
ct_uri = URIRef('https://clinicaltrials.gov/ct2/show/NCT01740427')
kg.add((ct_uri, RDF.type, SCHEMA.MedicalTrial))
kg.add((ct_uri, CT.hasID, Literal('NCT01740427')))
for i in paloma2:
    kg.add((ct_uri, CT.hasStudyType, Literal(paloma2['StudyType'])))
    if i == 'InclusionCriteria':
        for j in paloma2[i]:
            kg.add((ct_uri, CT.hasInclusionCriteria, Literal(j)))
    if i == 'ExclusionCriteria':
        for j in paloma2[i]:
            kg.add((ct_uri, CT.hasExclusionCriteria, Literal(j)))
    if i == 'Drugs':
        for j in paloma2[i]:
            k = URIRef(paloma2[i][j]['url'])
            kg.add((ct_uri, CT.hasDrug, k))
            kg.add((k, SCHEMA.name, Literal(j)))
            kg.add((k, SCHEMA.description, Literal(paloma2[i][j]['description'])))
    
    
    Group0 = paloma2["groups"][0].replace(' ',"_") + "_Group"
    Group1 = paloma2["groups"][1].replace(' ',"_") + "_Group"
    
    PFS_field = 'NCT04199078_PFS'
    kg.add((ct_uri, CT.hasPFS,CT[PFS_field]))
    kg.add((CT[PFS_field], CT.hasPFS,CT[PFS]))
    kg.add((CT[PFS], CT.hasGroup, Literal(Group0)))
    kg.add((CT[PFS], CT.hasType, Literal(paloma2["outcome_measurements"][0])))
    kg.add((CT[PFS], CT.hasUnit, Literal(paloma2["outcome_measurements"][1])))
    kg.add((CT[PFS], SCHEMA.number, Literal(paloma2['PFS'][0])))
    kg.add((CT[PFS_field], CT.hasPFS,CT[PFS1]))
    kg.add((CT[PFS1], CT.hasGroup, Literal(PFS1)))
    kg.add((CT[PFS1], CT.hasType, Literal(paloma2["outcome_measurements"][0])))
    kg.add((CT[PFS1], CT.hasUnit, Literal(paloma2["outcome_measurements"][1])))
    kg.add((CT[PFS1], SCHEMA.number, Literal(paloma2['PFS'][1])))
    
    treatment_field = 'NCT04199078_treamentDescription'
    treatment = Group0+"_Description"
    treatment1 = Group1+"_Description"
    kg.add((ct_uri, CT.hasTreatmentPlan,CT[treatment_field]))
    kg.add((CT[treatment_field], CT.hasTreatment,CT[treatment]))
    kg.add((CT[treatment], CT.hasGroup, Literal(Group0)))
    kg.add((CT[treatment], CT.hasdescription, Literal(paloma2["treatment"][0])))
    kg.add((CT[treatment_field], CT.hasTreatment,CT[treatment1]))
    kg.add((CT[treatment1], CT.hasGroup, Literal(Group1)))
    kg.add((CT[treatment1], CT.hasdescription, Literal(paloma2["treatment"][1])))
    
    nofP_field = 'NCT04199078_numberofPatients'
    numberofPatients = 'NCT04199078_numberofPatients_'+Group0
    numberofPatients1 = 'NCT04199078_numberofPatients_'+Group1
    kg.add((ct_uri, CT.hasNumberofPatients,CT[nofP_field]))
    kg.add((CT[nofP_field], CT.hasNumberofPatients,CT[numberofPatients]))
    kg.add((CT[numberofPatients], CT.hasGroup, Literal(Group0)))
    kg.add((CT[numberofPatients], SCHEMA.number, Literal(paloma2["numberOfPatients"][0])))
    kg.add((CT[numberofPatients], CT.hasUnit, Literal("person")))
    kg.add((CT[nofP_field], CT.hasNumberofPatients,CT[numberofPatients1]))
    kg.add((CT[numberofPatients1], CT.hasGroup, Literal(Group1)))
    kg.add((CT[numberofPatients1], SCHEMA.number, Literal(paloma2["numberOfPatients"][1])))
    kg.add((CT[numberofPatients1], CT.hasUnit, Literal("person")))
    
    aofP_field = 'NCT04199078_ageOfPatients'
    ageOfPatients = 'NCT04199078_ageOfPatients_'+Group0
    ageOfPatients1 = 'NCT04199078_ageOfPatients_'+Group1
    kg.add((ct_uri, CT.hasAge,CT[aofP_field]))
    kg.add((CT[aofP_field], CT.hasAgeofPatients,CT[ageOfPatients]))
    kg.add((CT[ageOfPatients], CT.hasGroup, Literal(Group0)))
    kg.add((CT[ageOfPatients], CT.hasType, Literal(paloma2["age_measurements"][0])))
    kg.add((CT[ageOfPatients], SCHEMA.number, Literal(paloma2["ageOfPatients"][0])))
    kg.add((CT[ageOfPatients], CT.hasUnit, Literal(paloma2["age_measurements"][1])))
    kg.add((CT[aofP_field], CT.hasAgeofPatients,CT[ageOfPatients1]))
    kg.add((CT[ageOfPatients1], CT.hasGroup, Literal(Group1)))
    kg.add((CT[ageOfPatients1], CT.hasType, Literal(paloma2["age_measurements"][0])))
    kg.add((CT[ageOfPatients1], SCHEMA.number, Literal(paloma2["ageOfPatients"][1])))
    kg.add((CT[ageOfPatients1], CT.hasUnit, Literal(paloma2["age_measurements"][1])))

#      'hazardRatio': '0.576',
#  'hasPercentageOfPatientsWithVisceralDisease': [48.2, 49.5],
#  'hasPercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy': [48, 49.1],
#  'hasPercentageOfPatientsWithNewlyDiagonosedAdavancedDisease': 37.2}
    PVD_field = 'NCT04199078_percentageofPatientWithVisceraDisease'
    PVD = 'NCT04199078_percentageofPatientWithVisceraDisease_'+Group0
    PVD1 = 'NCT04199078_percentageofPatientWithVisceraDisease_'+Group1
    kg.add((ct_uri, CT.hasPercentageOfPatientsWithVisceralDisease,CT[PVD_field]))
    kg.add((CT[PVD_field], CT.hasPercentageOfPatientsWithVisceralDisease,CT[PVD]))
    kg.add((CT[PVD], CT.hasGroup, Literal(Group0)))
    kg.add((CT[PVD], SCHEMA.number, Literal(paloma2["hasPercentageOfPatientsWithVisceralDisease"][0])))
    kg.add((CT[PVD], CT.hasUnit, Literal("percentage")))
    kg.add((CT[PVD_field], CT.hasPercentageOfPatientsWithVisceralDisease,CT[PVD1]))
    kg.add((CT[PVD1], CT.hasGroup, Literal(Group1)))
    kg.add((CT[PVD1], SCHEMA.number, Literal(paloma2["hasPercentageOfPatientsWithVisceralDisease"][1])))
    kg.add((CT[PVD1], CT.hasUnit, Literal("percentage")))
    
    PriorChemo_field = 'NCT04199078_percentageofPatientsHadReceivedPriorAdjuvantChemotherapy'
    PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy = 'NCT04199078_percentageofPatientsHadReceivedPriorAdjuvantChemotherapy_'+Group0
    PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy1 = 'NCT04199078_percentageofPatientsHadReceivedPriorAdjuvantChemotherapy_'+Group1
    kg.add((ct_uri, CT.hasPercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy,CT[PriorChemo_field]))
    kg.add((CT[PriorChemo_field], CT.hasPercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy,
            CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy]))
    kg.add((CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy], CT.hasGroup, Literal(Group0)))
    kg.add((CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy], SCHEMA.number, Literal(paloma2["hasPercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy"][0])))
    kg.add((CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy], CT.hasUnit, Literal("percentage")))
    kg.add((CT[PriorChemo_field], CT.hasPercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy,
            CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy1]))
    kg.add((CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy1], CT.hasGroup, Literal(Group1)))
    kg.add((CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy1], SCHEMA.number, Literal(paloma2["hasPercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy"][1])))
    kg.add((CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy1], CT.hasUnit, Literal("percentage")))    
    
    hazardRatio = paloma2["hazardRatio"]
    kg.add((ct_uri, CT.hasHazardRatio, CT['NCT04199078_hazardRatio']))
    kg.add((CT['NCT04199078_hazardRatio'], SCHEMA.number, Literal(paloma2["hazardRatio"])))
    kg.add((CT['NCT04199078_hazardRatio'], CT.hasConfidenceInterval, Literal("(2 sided)95% 0.463 to 0.718")))
    
    PercentageOfPatientsWithNewlyDiagonosedAdavancedDisease = str(paloma2["hasPercentageOfPatientsWithNewlyDiagonosedAdavancedDisease"])
    kg.add((ct_uri, CT.hasPercentageOfPatientsWithNewlyDiagonosedAdavancedDisease, CT['NCT04199078_PercentageOfPatientsWithNewlyDiagonosedAdavancedDisease']))
    kg.add((CT['NCT04199078_PercentageOfPatientsWithNewlyDiagonosedAdavancedDisease'], SCHEMA.number, Literal(paloma2["hasPercentageOfPatientsWithNewlyDiagonosedAdavancedDisease"])))
    kg.add((CT['NCT04199078_PercentageOfPatientsWithNewlyDiagonosedAdavancedDisease'], CT.hasUnit, Literal("percentage")))
    
kg.serialize('paloma-2.ttl', format="turtle")    
a
# kg.add((CT[drug], SCHEMA.Drug, SCHEMA.MedicalTrial))<jupyter_output><empty_output><jupyter_text>Generate knowledge graph<jupyter_code>g1 = Graph()
g1.parse("CT_model.ttl", format="turtle")

g2 = Graph()
g2.parse("paloma-2.ttl", format="turtle")

final_graph = g1 + g2

final_graph.serialize("CT_paloma2.ttl", format="turtle")<jupyter_output><empty_output><jupyter_text>Monaleesa2<jupyter_code>monaleesa2 = dict()
studytype_url = 'https://clinicaltrials.gov/api/query/field_values?expr=monaleesa2&field=StudyType&fmt=json'
studytype = requests.get(studytype_url).json()['FieldValuesResponse']['FieldValues'][0]['FieldValue']
monaleesa2['StudyType'] = studytype
criteria_url = 'https://clinicaltrials.gov/api/query/full_studies?expr=monaleesa2&min_rnk=1&max_rnk=&fmt=json'
criteria = requests.get(criteria_url).json()['FullStudiesResponse']['FullStudies'][0]['Study']['ProtocolSection']['EligibilityModule']['EligibilityCriteria']

b = criteria.split('\n')
inclusion = []
exclusion = []
for i in range(0,len(b)):
    if b[i] == 'Inclusion Criteria:':
        for j in range(i+1,len(b)):
            if b[j] == 'Exclusion Criteria:':
                break
            if b[j] != '':
                inclusion.append(b[j])
    if b[i] == 'Exclusion Criteria:':
        for j in range(i+1,len(b)):
            if b[j] != '':
                exclusion.append(b[j])
monaleesa2['InclusionCriteria'] = inclusion
monaleesa2['ExclusionCriteria'] = exclusion

drug_url = 'https://clinicaltrials.gov/api/query/study_fields?expr=monaleesa-2&fields=ArmGroupInterventionName&min_rnk=1&max_rnk=&fmt=json'
drug = requests.get(drug_url).json()['StudyFieldsResponse']['StudyFields'][0]['ArmGroupInterventionName']
drug
drugs = [i.strip("Drug:").strip("'").strip() for i in drug] 
drugs2 = set()
for i in drugs:
    for j in i.split(" "):
        drugs2.add(j)
# drugs = list(dict.fromkeys(drugs))
drugs2
API_ENDPOINT = 'https://en.wikipedia.org/w/api.php'
drugs_with_des = {}
for i in drugs2:
    drugs_with_des[i] = {}
    query = i
    params = {
        'action':'query',
        'list':'search',
        'srsearch':query,
        'format':'json',
    }
    drugs_with_des[i]['url'] = requests.get(API_ENDPOINT,params=params).url
    drugs_with_des[i]['description'] = requests.get(API_ENDPOINT,params=params).json()\
    ['query']['search'][0]['snippet']
drugs_with_des
monaleesa2['Drugs'] = drugs_with_des
outcome_url = "https://clinicaltrials.gov/api/query/study_fields?expr=monaleesa2&fields=OutcomeMeasurementValue&min_rnk=1&max_rnk=&fmt=json"
outcome = requests.get(outcome_url).json()["StudyFieldsResponse"]["StudyFields"][0]["OutcomeMeasurementValue"][:2]

treatment_url = "https://clinicaltrials.gov/api/query/study_fields?expr=monaleesa2&fields=ArmGroupDescription&min_rnk=1&max_rnk=&fmt=json"
treatment = requests.get(treatment_url).json()["StudyFieldsResponse"]["StudyFields"][0]["ArmGroupDescription"]

numberOfPatients_url = "https://clinicaltrials.gov/api/query/study_fields?expr=monaleesa2&fields=BaselineDenomCountValue&min_rnk=1&max_rnk=&fmt=json"
numberOfPatients = requests.get(numberOfPatients_url).json()["StudyFieldsResponse"]["StudyFields"][0]["BaselineDenomCountValue"]

ageOfPatients_url = "https://clinicaltrials.gov/api/query/study_fields?expr=monaleesa2&fields=BaselineMeasurementValue&min_rnk=1&max_rnk=&fmt=json"
ageOfPatients = requests.get(ageOfPatients_url).json()["StudyFieldsResponse"]["StudyFields"][0]["BaselineMeasurementValue"][:3]
ageOfPatients

hazardRatio_url = "https://clinicaltrials.gov/api/query/study_fields?expr=monaleesa2&fields=OutcomeAnalysisParamValue&min_rnk=1&max_rnk=&fmt=json"
hazardRatio = requests.get(hazardRatio_url).json()["StudyFieldsResponse"]["StudyFields"][0]["OutcomeAnalysisParamValue"][0]
hazardRatio
# monaleesa2["outcome"] = outcome
# monaleesa2["treatment"] = treatment
# monaleesa2["numberOfPatients"] = numberOfPatients
# monaleesa2["ageOfPatients"] = ageOfPatients
# monaleesa2["hazardRatio"] = hazardRatio
url = "https://clinicaltrials.gov/ct2/show/results/NCT01958021?view=results"
html = urlopen(url)
soup = BS(html, 'html.parser')

monaleesa2["numberOfPatients"] = []
monaleesa2["numberOfPatients"].append(soup.find_all("td", class_="de-numValue_popFlowCell")[0].text.strip("\r").strip("\n").strip())
monaleesa2["numberOfPatients"].append(soup.find_all("td", class_="de-numValue_popFlowCell")[1].text.strip("\r").strip("\n").strip())
monaleesa2["outcome"] = []
monaleesa2["outcome"].append(soup.find_all("td", class_="de-numValue_outcomeDataCell")[2].text.strip('\n').strip().split(" ")[0])
monaleesa2["outcome"].append(soup.find_all("td", class_="de-numValue_outcomeDataCell")[3].text.strip('\n').strip().split(" ")[0])
monaleesa2["treatment"] = []
monaleesa2["treatment"].append(soup.find_all("td",class_="de-outcomeLabelCell")[7].text.strip('\n').strip('\r').strip())
monaleesa2["treatment"].append(soup.find_all("td",class_="de-outcomeLabelCell")[8].text.strip('\n').strip('\r').strip())

monaleesa2["hazardRatio"] = soup.find_all("td",class_="de-outcomeDataCell")[15].text.strip('\n').strip('\r').strip()
monaleesa2["hasPercentageOfPatientsWithVisceralDisease"] = 77.5
monaleesa2["hasPercentageOfPatientsWithVisceralDisease"] = [48.2,49.5]
monaleesa2["hasPercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy"] = [48,49.1]
monaleesa2["hasPercentageOfPatientsWithNewlyDiagonosedAdavancedDisease"] = 37.2
url = "https://clinicaltrials.gov/ct2/show/results/NCT01958021?view=results"
response = urlopen(url)
soup = BS(response, 'html.parser')
baseline = soup.find_all("th",class_="de-popFlowLabelCell")
groups = []
for i in range(0,len(baseline)):
    groups.append(baseline[i].text.strip("\n").strip("\r").strip())
    number_of_patients.append(soup.find_all("td",class_="de-numValue_baselineDataCell")[i].text.strip("\n").strip("\r").strip())
monaleesa2["groups"] = groups
dom = etree.HTML(str(soup))
monaleesa2['PFS'] = [dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[6]/td[1]/div[1]/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''),
                  dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[6]/td[2]/div[1]/text()')[0].strip('\n').strip().strip('\n').strip()]
monaleesa2["outcome_measurements"] = [dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[5]/th/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''),
                             dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[5]/th/div/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')]
monaleesa2["age_measurements"] = [dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[7]/th[1]/div/div[1]/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''),
                             dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[7]/th[1]/div/div[2]/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')]
monaleesa2["ageOfPatients"] = [dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[9]/td[2]/span/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''),
                               dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[9]/td[3]/span/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')]

monaleesa2

kg = Graph()
kg.bind('Clinical_Trials', CT)
kg.bind('schema',SCHEMA)
ct_uri = URIRef('https://clinicaltrials.gov/ct2/show/NCT01958021')
kg.add((ct_uri, RDF.type, SCHEMA.MedicalTrial))
kg.add((ct_uri, CT.hasID, Literal('NCT01958021')))
for i in monaleesa2:
    kg.add((ct_uri, CT.hasStudyType, Literal(monaleesa2['StudyType'])))
    if i == 'InclusionCriteria':
        for j in monaleesa2[i]:
            kg.add((ct_uri, CT.hasInclusionCriteria, Literal(j)))
    if i == 'ExclusionCriteria':
        for j in monaleesa2[i]:
            kg.add((ct_uri, CT.hasExclusionCriteria, Literal(j)))
    if i == 'Drugs':
        for j in monaleesa2[i]:
            k = URIRef(monaleesa2[i][j]['url'])
            kg.add((ct_uri, CT.hasDrug, k))
            kg.add((k, SCHEMA.name, Literal(j)))
            kg.add((k, SCHEMA.description, Literal(monaleesa2[i][j]['description'])))
    
    
    Group = monaleesa2["groups"][0].replace(' ',"_") + "_Group"
    Group1 = monaleesa2["groups"][1].replace(' ',"_") + "_Group"
    
    PFS_field = 'NCT01958021_PFS'
    kg.add((ct_uri, CT.hasPFS,CT[PFS_field]))
    kg.add((CT[PFS_field], CT.hasPFS,CT[PFS]))
    kg.add((CT[PFS], CT.hasGroup, Literal(Group)))
    kg.add((CT[PFS], CT.hasType, Literal(monaleesa2["outcome_measurements"][0])))
    kg.add((CT[PFS], CT.hasUnit, Literal(monaleesa2["outcome_measurements"][1])))
    kg.add((CT[PFS], SCHEMA.number, Literal(monaleesa2['PFS'][0])))
    kg.add((CT[PFS_field], CT.hasPFS,CT[PFS1]))
    kg.add((CT[PFS1], CT.hasGroup, Literal(Group1)))
    kg.add((CT[PFS1], CT.hasType, Literal(monaleesa2["outcome_measurements"][0])))
    kg.add((CT[PFS1], CT.hasUnit, Literal(monaleesa2["outcome_measurements"][1])))
    kg.add((CT[PFS1], SCHEMA.number, Literal(monaleesa2['PFS'][1])))
    
    treatment_field = 'NCT01958021_treamentDescription'
    treatment = Group+"_Description"
    treatment1 = Group1+"_Description"
    kg.add((ct_uri, CT.hasTreatmentPlan,CT[treatment_field]))
    kg.add((CT[treatment_field], CT.hasTreatment,CT[treatment]))
    kg.add((CT[treatment], CT.hasGroup, Literal(Group)))
    kg.add((CT[treatment], CT.hasdescription, Literal(monaleesa2["treatment"][0])))
    kg.add((CT[treatment_field], CT.hasTreatment,CT[treatment1]))
    kg.add((CT[treatment1], CT.hasGroup, Literal(Group1)))
    kg.add((CT[treatment1], CT.hasdescription, Literal(monaleesa2["treatment"][1])))
    
    nofP_field = 'NCT01958021_numberofPatients'
    numberofPatients = 'NCT01958021_numberofPatients_'+Group
    numberofPatients1 = 'NCT01958021_numberofPatients_'+Group1
    kg.add((ct_uri, CT.hasNumberofPatients,CT[nofP_field]))
    kg.add((CT[nofP_field], CT.hasNumberofPatients,CT[numberofPatients]))
    kg.add((CT[numberofPatients], CT.hasGroup, Literal(Group)))
    kg.add((CT[numberofPatients], SCHEMA.number, Literal(monaleesa2["numberOfPatients"][0])))
    kg.add((CT[numberofPatients], CT.hasUnit, Literal("person")))
    kg.add((CT[nofP_field], CT.hasNumberofPatients,CT[numberofPatients1]))
    kg.add((CT[numberofPatients1], CT.hasGroup, Literal(Group1)))
    kg.add((CT[numberofPatients1], SCHEMA.number, Literal(monaleesa2["numberOfPatients"][1])))
    kg.add((CT[numberofPatients1], CT.hasUnit, Literal("person")))
    
    aofP_field = 'NCT01958021_ageOfPatients'
    ageOfPatients = 'NCT01958021_ageOfPatients_'+Group
    ageOfPatients1 = 'NCT01958021_ageOfPatients_'+Group1
    kg.add((ct_uri, CT.hasAge,CT[aofP_field]))
    kg.add((CT[aofP_field], CT.hasAgeofPatients,CT[ageOfPatients]))
    kg.add((CT[ageOfPatients], CT.hasGroup, Literal(Group)))
    kg.add((CT[ageOfPatients], CT.hasType, Literal(monaleesa2["age_measurements"][0])))
    kg.add((CT[ageOfPatients], SCHEMA.number, Literal(monaleesa2["ageOfPatients"][0])))
    kg.add((CT[ageOfPatients], CT.hasUnit, Literal(monaleesa2["age_measurements"][1])))
    kg.add((CT[aofP_field], CT.hasAgeofPatients,CT[ageOfPatients1]))
    kg.add((CT[ageOfPatients1], CT.hasGroup, Literal(Group1)))
    kg.add((CT[ageOfPatients1], CT.hasType, Literal(monaleesa2["age_measurements"][0])))
    kg.add((CT[ageOfPatients1], SCHEMA.number, Literal(monaleesa2["ageOfPatients"][1])))
    kg.add((CT[ageOfPatients1], CT.hasUnit, Literal(monaleesa2["age_measurements"][1])))

#      'hazardRatio': '0.576',
#  'hasPercentageOfPatientsWithVisceralDisease': [48.2, 49.5],
#  'hasPercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy': [48, 49.1],
#  'hasPercentageOfPatientsWithNewlyDiagonosedAdavancedDisease': 37.2}
    PVD_field = 'NCT01958021_percentageofPatientWithVisceraDisease'
    PVD = 'NCT01958021_percentageofPatientWithVisceraDisease_'+Group
    PVD1 = 'NCT01958021_percentageofPatientWithVisceraDisease_'+Group1
    kg.add((ct_uri, CT.hasPercentageOfPatientsWithVisceralDisease,CT[PVD_field]))
    kg.add((CT[PVD_field], CT.hasPercentageOfPatientsWithVisceralDisease,CT[PVD]))
    kg.add((CT[PVD], CT.hasGroup, Literal(Group)))
    kg.add((CT[PVD], SCHEMA.number, Literal(monaleesa2["hasPercentageOfPatientsWithVisceralDisease"][0])))
    kg.add((CT[PVD], CT.hasUnit, Literal("percentage")))
    kg.add((CT[PVD_field], CT.hasPercentageOfPatientsWithVisceralDisease,CT[PVD1]))
    kg.add((CT[PVD1], CT.hasGroup, Literal(Group1)))
    kg.add((CT[PVD1], SCHEMA.number, Literal(monaleesa2["hasPercentageOfPatientsWithVisceralDisease"][1])))
    kg.add((CT[PVD1], CT.hasUnit, Literal("percentage")))
    
    PriorChemo_field = 'NCT01958021_percentageofPatientsHadReceivedPriorAdjuvantChemotherapy'
    PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy = 'NCT01958021_percentageofPatientsHadReceivedPriorAdjuvantChemotherapy_'+Group
    PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy1 = 'NCT01958021_percentageofPatientsHadReceivedPriorAdjuvantChemotherapy_'+Group1
    kg.add((ct_uri, CT.hasPercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy,CT[PriorChemo_field]))
    kg.add((CT[PriorChemo_field], CT.hasPercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy,
            CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy]))
    kg.add((CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy], CT.hasGroup, Literal(Group)))
    kg.add((CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy], SCHEMA.number, Literal(monaleesa2["hasPercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy"][0])))
    kg.add((CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy], CT.hasUnit, Literal("percentage")))
    kg.add((CT[PriorChemo_field], CT.hasPercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy,
            CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy1]))
    kg.add((CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy1], CT.hasGroup, Literal(Group1)))
    kg.add((CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy1], SCHEMA.number, Literal(monaleesa2["hasPercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy"][1])))
    kg.add((CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy1], CT.hasUnit, Literal("percentage")))    
    
    hazardRatio = monaleesa2["hazardRatio"]
    kg.add((ct_uri, CT.hasHazardRatio, CT['NCT01958021_hazardRatio']))
    kg.add((CT['NCT01958021_hazardRatio'], SCHEMA.number, Literal(monaleesa2["hazardRatio"])))
    kg.add((CT['NCT01958021_hazardRatio'], CT.hasConfidenceInterval, Literal("(2 sided)95% 0.463 to 0.718")))
    
    PercentageOfPatientsWithNewlyDiagonosedAdavancedDisease = str(monaleesa2["hasPercentageOfPatientsWithNewlyDiagonosedAdavancedDisease"])
    kg.add((ct_uri, CT.hasPercentageOfPatientsWithNewlyDiagonosedAdavancedDisease, CT['NCT01958021_PercentageOfPatientsWithNewlyDiagonosedAdavancedDisease']))
    kg.add((CT['NCT01958021_PercentageOfPatientsWithNewlyDiagonosedAdavancedDisease'], SCHEMA.number, Literal(monaleesa2["hasPercentageOfPatientsWithNewlyDiagonosedAdavancedDisease"])))
    kg.add((CT['NCT01958021_PercentageOfPatientsWithNewlyDiagonosedAdavancedDisease'], CT.hasUnit, Literal("percentage")))
    
kg.serialize('monaleesa-2.ttl', format="turtle")     
    
kg = Graph()
kg.bind('Clinical_Trials', CT)
kg.bind('schema',SCHEMA)
ct_uri = URIRef('https://clinicaltrials.gov/ct2/show/NCT00984399')
kg.add((ct_uri, CT['hasNoResults'], Literal("No Results")))
kg.serialize('new.ttl', format="turtle")  
g1 = Graph()
g1.parse("NCT00984399.ttl", format="turtle")

g2 = Graph()
g2.parse("new.ttl", format="turtle")

final_graph = g1 + g2

final_graph.serialize("new_NCT00984399.ttl", format="turtle")

kg = Graph()
kg.bind('Clinical_Trials', CT)
kg.bind('schema',SCHEMA)
ct_uri = URIRef('https://clinicaltrials.gov/ct2/show/NCT01478477')
kg.add((ct_uri, CT['hasNoResults'], Literal("No Results")))
kg.serialize('new.ttl', format="turtle")  
g1 = Graph()
g1.parse("NCT01478477.ttl", format="turtle")

g2 = Graph()
g2.parse("new.ttl", format="turtle")

final_graph = g1 + g2

final_graph.serialize("new_NCT01478477.ttl", format="turtle")
kg = Graph()
kg.bind('Clinical_Trials', CT)
kg.bind('schema',SCHEMA)
ct_uri = URIRef('https://clinicaltrials.gov/ct2/show/NCT01676753')
kg.add((ct_uri, CT['hasNoResults'], Literal("No Results")))
kg.serialize('new.ttl', format="turtle")  
g1 = Graph()
g1.parse("NCT01676753.ttl", format="turtle")

g2 = Graph()
g2.parse("new.ttl", format="turtle")

final_graph = g1 + g2

final_graph.serialize("new_graphs/new_NCT01676753.ttl", format="turtle")
kg = Graph()
kg.bind('Clinical_Trials', CT)
kg.bind('schema',SCHEMA)
ct_uri = URIRef('https://clinicaltrials.gov/ct2/show/NCT01791478')
kg.add((ct_uri, CT['hasNoResults'], Literal("No Results")))
kg.serialize('new.ttl', format="turtle")  
g1 = Graph()
g1.parse("NCT01791478.ttl", format="turtle")

g2 = Graph()
g2.parse("new.ttl", format="turtle")

final_graph = g1 + g2

final_graph.serialize("new_graphs/new_NCT01791478.ttl", format="turtle")
kg = Graph()
kg.bind('Clinical_Trials', CT)
kg.bind('schema',SCHEMA)
ct_uri = URIRef('https://clinicaltrials.gov/ct2/show/NCT01824836')
kg.add((ct_uri, CT['hasNoResults'], Literal("No Results")))
kg.serialize('new.ttl', format="turtle")  
g1 = Graph()
g1.parse("NCT01824836.ttl", format="turtle")

g2 = Graph()
g2.parse("new.ttl", format="turtle")

final_graph = g1 + g2

final_graph.serialize("new_graphs/new_NCT01824836.ttl", format="turtle")
kg = Graph()
kg.bind('Clinical_Trials', CT)
kg.bind('schema',SCHEMA)
ct_uri = URIRef('https://clinicaltrials.gov/ct2/show/NCT02203513')
kg.add((ct_uri, CT['hasNoResults'], Literal("No Results")))
kg.serialize('new.ttl', format="turtle")  
g1 = Graph()
g1.parse("NCT02203513.ttl", format="turtle")

g2 = Graph()
g2.parse("new.ttl", format="turtle")

final_graph = g1 + g2

final_graph.serialize("new_graphs/new_NCT02203513.ttl", format="turtle")
kg = Graph()
kg.bind('Clinical_Trials', CT)
kg.bind('schema',SCHEMA)
ct_uri = URIRef('https://clinicaltrials.gov/ct2/show/NCT02379247')
kg.add((ct_uri, CT['hasNoResults'], Literal("Results not posted on ClinicalTrials.gov")))
kg.serialize('new.ttl', format="turtle")  
g1 = Graph()
g1.parse("NCT02379247.ttl", format="turtle")

g2 = Graph()
g2.parse("new.ttl", format="turtle")

final_graph = g1 + g2

final_graph.serialize("new_graphs/new_NCT02379247.ttl", format="turtle")
kg = Graph()
kg.bind('Clinical_Trials', CT)
kg.bind('schema',SCHEMA)
ct_uri = URIRef('https://clinicaltrials.gov/ct2/show/NCT02530489')
kg.add((ct_uri, CT['hasNoResults'], Literal("No Results")))
kg.serialize('new.ttl', format="turtle")  
g1 = Graph()
g1.parse("NCT02530489.ttl", format="turtle")

g2 = Graph()
g2.parse("new.ttl", format="turtle")

final_graph = g1 + g2

final_graph.serialize("new_graphs/new_NCT02530489.ttl", format="turtle")
kg = Graph()
kg.bind('Clinical_Trials', CT)
kg.bind('schema',SCHEMA)
ct_uri = URIRef('https://clinicaltrials.gov/ct2/show/NCT02657343')
kg.add((ct_uri, CT['hasNoResults'], Literal("No Results")))
kg.serialize('new.ttl', format="turtle")  
g1 = Graph()
g1.parse("NCT02657343.ttl", format="turtle")

g2 = Graph()
g2.parse("new.ttl", format="turtle")

final_graph = g1 + g2

final_graph.serialize("new_graphs/new_NCT02657343.ttl", format="turtle")
kg = Graph()
kg.bind('Clinical_Trials', CT)
kg.bind('schema',SCHEMA)
ct_uri = URIRef('https://clinicaltrials.gov/ct2/show/NCT02672475')
kg.add((ct_uri, CT['hasNoResults'], Literal("No Results")))
kg.serialize('new.ttl', format="turtle")  
g1 = Graph()
g1.parse("NCT02672475.ttl", format="turtle")

g2 = Graph()
g2.parse("new.ttl", format="turtle")

final_graph = g1 + g2

final_graph.serialize("new_graphs/new_NCT02672475.ttl", format="turtle")
kg = Graph()
kg.bind('Clinical_Trials', CT)
kg.bind('schema',SCHEMA)
ct_uri = URIRef('https://clinicaltrials.gov/ct2/show/NCT02719691')
kg.add((ct_uri, CT['hasNoResults'], Literal("No Results")))
kg.serialize('new.ttl', format="turtle")  
g1 = Graph()
g1.parse("NCT02719691.ttl", format="turtle")

g2 = Graph()
g2.parse("new.ttl", format="turtle")

final_graph = g1 + g2

final_graph.serialize("new_graphs/new_NCT02719691.ttl", format="turtle")
kg = Graph()
kg.bind('Clinical_Trials', CT)
kg.bind('schema',SCHEMA)
ct_uri = URIRef('https://clinicaltrials.gov/ct2/show/NCT03219476')
kg.add((ct_uri, CT['hasNoResults'], Literal("No Results")))
kg.serialize('new.ttl', format="turtle")  
g1 = Graph()
g1.parse("NCT03219476.ttl", format="turtle")

g2 = Graph()
g2.parse("new.ttl", format="turtle")

final_graph = g1 + g2

final_graph.serialize("new_graphs/new_NCT03219476.ttl", format="turtle")
kg = Graph()
kg.bind('Clinical_Trials', CT)
kg.bind('schema',SCHEMA)
ct_uri = URIRef('https://clinicaltrials.gov/ct2/show/NCT03874325')
kg.add((ct_uri, CT['hasNoResults'], Literal("No Results")))
kg.serialize('new.ttl', format="turtle")  
g1 = Graph()
g1.parse("NCT03874325.ttl", format="turtle")

g2 = Graph()
g2.parse("new.ttl", format="turtle")

final_graph = g1 + g2

final_graph.serialize("new_graphs/new_NCT03874325.ttl", format="turtle")
url = "https://clinicaltrials.gov/ct2/show/results/NCT03093350?view=results"
response = urlopen(url)
soup = BS(response, 'html.parser')
dom = etree.HTML(str(soup))
## age: td[2]
a = [str(dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[9]/td[2]/span/text()')[0])
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')]
## outcome: td[1]
b = [dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[6]/td[1]/div[1]/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')]
b
url = "https://clinicaltrials.gov/ct2/show/results/NCT03093350?view=results"
response = urlopen(url)
soup = BS(response, 'html.parser')
baseline = soup.find_all("th",class_="de-popFlowLabelCell")
groups = []
number_of_patients = []
for i in range(0,len(baseline)):
    groups.append(baseline[i].text.strip("\n").strip("\r").strip())
    number_of_patients.append(soup.find_all("td",class_="de-numValue_baselineDataCell")[i].text.strip("\n").strip("\r").strip())

response = urlopen(url)
soup = BS(response, 'html.parser')
dom = etree.HTML(str(soup))

NCT03093350 = dict()
NCT03093350["numberOfPatients"] = []
NCT03093350["numberOfPatients"].append(soup.find_all("td", class_="de-numValue_popFlowCell")[0].text.strip("\r").strip("\n").strip())
NCT03093350["numberOfPatients"].append(soup.find_all("td", class_="de-numValue_popFlowCell")[1].text.strip("\r").strip("\n").strip())
NCT03093350["treatment"] = dom.xpath('//*[@id="EXPAND-armGroupDescriptionRow-baseline"]/td[2]/text()')
NCT03093350["ageOfPatients"] = [dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[9]/td[2]/span/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')]
# NCT03093350["hazardRatio"] = soup.find_all("td",class_="de-outcomeDataCell")[15].text.strip('\n').strip('\r').strip()

NCT03093350['outcome'] = [dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[6]/td[1]/div[1]/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')]
NCT03093350["outcome_measurements"] = [dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[5]/th/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''),
                             dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[5]/th/div/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')]
NCT03093350["age_measurements"] = [dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[7]/th[1]/div/div[1]/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''),
                             dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[7]/th[1]/div/div[2]/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')]
NCT03093350


def crawling_data(ID):
    url = 'https://clinicaltrials.gov/ct2/show/results/'+ID+'?view=results'
    
    response = urlopen(url)
    soup = BS(response, 'html.parser')
    baseline = soup.find_all("th",class_="de-popFlowLabelCell")
    response = urlopen(url)
    soup = BS(response, 'html.parser')
    dom = etree.HTML(str(soup))
    output = dict()
    output['groups'] = []
    output["treatment"] = []
    output["ageOfPatients"] = []
    output["outcome"] = []
    output["outcome_measurements"] = []
    output["age_measurements"] = []
    output["outcome_description"] = []

    if (len(baseline)>0):
        groups = []
        for i in range(0,len(baseline)):
            groups.append(baseline[i].text.strip("\n").strip("\r").strip())
            number_of_patients.append(soup.find_all("td",class_="de-numValue_baselineDataCell")[i].text.strip("\n").strip("\r").strip())
        
        output['groups'] = groups
        output["numberOfPatients"] = []
        if(len(soup.find_all("td", class_="de-numValue_popFlowCell"))>0):          
            for i in range(0,len(baseline)):
                output["numberOfPatients"].append(soup.find_all("td", class_="de-numValue_popFlowCell")[i].text.strip("\r").strip("\n").strip())

 
        treatment = dom.xpath('//*[@id="EXPAND-armGroupDescriptionRow-baseline"]/td[2]/text()')
        if(len(treatment)>0):
            print('1')
            output["treatment"].append(dom.xpath('//*[@id="EXPAND-armGroupDescriptionRow-baseline"]/td[' + str(i+2)+ ']/text()')[0]
                                       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''))
        else:
            print('2')
            treatment = dom.xpath('//*[@id="EXPAND-armGroupDescriptionRow-baseline"]/td[2]/p[1]/text()')
            if(len(treatment)>0):
                print(treatment)
                output["treatment"].append(dom.xpath('//*[@id="EXPAND-armGroupDescriptionRow-baseline"]/td['+str(i+2)+']/p[1]/text()')[0]
                                           .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''))
          
        output["age_measurements"] = [dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[7]/th[1]/div/div[1]/text()'),
                                     dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[7]/th[1]/div/div[2]/text()')]
        if(len(output["age_measurements"])>0):
            output["age_measurements"] = [dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[7]/th[1]/div/div[1]/text()')
                                          [0].strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''),
                                     dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[7]/th[1]/div/div[2]/text()')
                                         [0].strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')]

        if('count' in output["age_measurements"][0].lower()):
            print("need to check count")
        else:
            
            age = dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[9]/td[2]/span/text()')
    #         //*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[9]/td[2]/div[1]
            if(len(age)>0):
                for i in range(0,len(baseline)):
                    output["ageOfPatients"].append(dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[9]/td[' + str(i+2) + ']/span/text()')
                                                   [0].strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''))
            else:
                age = dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[9]/td[2]/div[1]/text()') 
                for i in range(0,len(baseline)):
                    output["ageOfPatients"].append(dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[9]/td['+str(i+2)+']/div[1]/text()')
                                                   [0].strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''))
    # //*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[9]/td[2]/div[2]
    # //*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[10]/td[2]/div[1]

        outcome = dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[6]/td[1]/div[1]/text()')
        
        if(len(outcome) > 0):
            for i in range(0,len(baseline)):
                output["outcome"].append(dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[6]/td[' + str(i+1) + ']/div[1]/text()')
                                         [0].strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''))
        else:
            output["outcome"] = len(outcome)
        output["outcome_measurements"] = [dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[5]/th/text()'),
                                     dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[5]/th/div/text()')]
        if(len(output["outcome_measurements"]) > 0):
            output["outcome_measurements"] = [dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[5]/th/text()')
                                              [0].strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''),
                                     dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[5]/th/div/text()')
                                             [0].strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')]
        outcome_description = dom.xpath('//*[@id="tab-body"]/div[1]/div[4]/div[2]/fieldset[1]/div/table/tbody/tr[1]/td/text()')
        if (len(outcome_description)>0):
            print('0')
            output["outcome_description"] = dom.xpath('//*[@id="tab-body"]/div[1]/div[4]/div[2]/fieldset[1]/div/table/tbody/tr[1]/td/text()')
            [0].strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')
    return output




url = 'https://clinicaltrials.gov/ct2/show/results/NCT03093350?view=results'
    
response = urlopen(url)
soup = BS(response, 'html.parser')
baseline = soup.find_all("th",class_="de-popFlowLabelCell")
response = urlopen(url)
soup = BS(response, 'html.parser')
dom = etree.HTML(str(soup))
treatment = dom.xpath('//*[@id="EXPAND-armGroupDescriptionRow-baseline"]/td[2]//text()')
description = dom.xpath('//*[@id="tab-body"]/div[1]/div[4]/div[2]/fieldset[1]/div/table/tbody/tr[1]/td/text()')
description


print(crawling_data('NCT02028221'))

print(crawling_data('NCT00003042'))

print(crawling_data('NCT02278120'))
print(crawling_data('NCT01035099'))
print(crawling_data('NCT03093350'))
paloma2<jupyter_output><empty_output><jupyter_text>Complished: 1. Takes ~30 seconds to extract all information and create a knowledge graph 2. Checked number of columns(groups) before extracting information 3. Changed arm/group title from treatment/control groups to their actual names 4. Added types and units of age and outcome description directly from the table 5. Solve mistakes by different structure:   i : //*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[9]/td[2]/div[1] with //*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[9]/td[2]/span   ii: count of participants in each age group vs median/mean age (to-do: there will be new properties in knowledge graph about number of patients in each group, currently left blank) To-do: 1. Extract arm group Description from all p of a td: https://clinicaltrials.gov/ct2/show/results/NCT03093350 2. Can't get text directly inside td: 3350<jupyter_code>NCT03093350 = dict()
studytype_url = 'https://clinicaltrials.gov/api/query/study_fields?expr=NCT03093350&fields=studytype&min_rnk=1&max_rnk=&fmt=json'
studytype = requests.get(studytype_url).json()['StudyFieldsResponse']['StudyFields'][0]['StudyType'][0]
NCT03093350['StudyType'] = studytype
criteria_url = 'https://clinicaltrials.gov/api/query/full_studies?expr=NCT04176354&min_rnk=1&max_rnk=&fmt=json'
criteria = requests.get(criteria_url).json()['FullStudiesResponse']['FullStudies'][0]['Study']['ProtocolSection']['EligibilityModule']['EligibilityCriteria']

b = criteria.split('\n')
inclusion = []
exclusion = []
for i in range(0,len(b)):
    if b[i] == 'Inclusion Criteria:':
        for j in range(i+1,len(b)):
            if b[j] == 'Exclusion Criteria:':
                break
            if b[j] != '':
                inclusion.append(b[j])
    if b[i] == 'Exclusion Criteria:':
        for j in range(i+1,len(b)):
            if b[j] != '':
                exclusion.append(b[j])
NCT03093350['InclusionCriteria'] = inclusion
NCT03093350['ExclusionCriteria'] = exclusion

drug_url = 'https://clinicaltrials.gov/api/query/study_fields?expr=NCT04176354&fields=ArmGroupInterventionName&min_rnk=1&max_rnk=&fmt=json'
drug = requests.get(drug_url).json()['StudyFieldsResponse']['StudyFields'][0]['ArmGroupInterventionName']
drug
drugs = [i.strip("Drug:").strip("'").strip() for i in drug] 
drugs2 = set()
for i in drugs:
    for j in i.split(" "):
        drugs2.add(j)
# drugs = list(dict.fromkeys(drugs))
drugs2.remove('an')
drugs2.remove('+')
drugs2.remove('aromatase')
drugs2.remove('inhibito')

API_ENDPOINT = 'https://en.wikipedia.org/w/api.php'
drugs_with_des = {}
for i in drugs2:
    drugs_with_des[i] = {}
    query = i
    params = {
        'action':'query',
        'list':'search',
        'srsearch':query,
        'format':'json',
    }
    drugs_with_des[i]['url'] = requests.get(API_ENDPOINT,params=params).url
    drugs_with_des[i]['description'] = requests.get(API_ENDPOINT,params=params).json()\
    ['query']['search'][0]['snippet']
drugs_with_des
NCT03093350['Drugs'] = drugs_with_des



url = "https://clinicaltrials.gov/ct2/show/results/NCT03093350?view=results"
response = urlopen(url)
soup = BS(response, 'html.parser')
baseline = soup.find_all("th",class_="de-popFlowLabelCell")
groups = []
number_of_patients = []
for i in range(0,len(baseline)):
    groups.append(baseline[i].text.strip("\n").strip("\r").strip())
    number_of_patients.append(soup.find_all("td",class_="de-numValue_baselineDataCell")[i].text.strip("\n").strip("\r").strip())

response = urlopen(url)
soup = BS(response, 'html.parser')
dom = etree.HTML(str(soup))


monaleesa2["numberOfPatients"] = []
monaleesa2["numberOfPatients"].append(soup.find_all("td", class_="de-numValue_popFlowCell")[0].text.strip("\r").strip("\n").strip())
monaleesa2["numberOfPatients"].append(soup.find_all("td", class_="de-numValue_popFlowCell")[1].text.strip("\r").strip("\n").strip())
monaleesa2["outcome"] = []
monaleesa2["outcome"].append(soup.find_all("td", class_="de-numValue_outcomeDataCell")[2].text.strip('\n').strip().split(" ")[0])
monaleesa2["outcome"].append(soup.find_all("td", class_="de-numValue_outcomeDataCell")[3].text.strip('\n').strip().split(" ")[0])
monaleesa2["treatment"] = []
monaleesa2["treatment"].append(soup.find_all("td",class_="de-outcomeLabelCell")[7].text.strip('\n').strip('\r').strip())
monaleesa2["treatment"].append(soup.find_all("td",class_="de-outcomeLabelCell")[8].text.strip('\n').strip('\r').strip())
monaleesa2["ageOfPatients"] = []
monaleesa2["ageOfPatients"].append(soup.find_all("td",class_="de-numValue_baselineDataCell")[11].text.strip('\n').strip('\r').strip().split(" ")[0])
monaleesa2["ageOfPatients"].append(soup.find_all("td",class_="de-numValue_baselineDataCell")[12].text.strip('\n').strip('\r').strip().split(" ")[0])
monaleesa2["hazardRatio"] = soup.find_all("td",class_="de-outcomeDataCell")[15].text.strip('\n').strip('\r').strip()

monaleesa2['PFS'] = [dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[6]/td[1]/div[1]/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''),
                  dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[6]/td[2]/div[1]/text()')[0].strip('\n').strip().strip('\n').strip()]
monaleesa2["outcome_measurements"] = [dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[5]/th/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''),
                             dom.xpath('//*[@id="EXPAND-outcome-data-1"]/table/tbody/tr[5]/th/div/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')]
monaleesa2["age_measurements"] = [dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[7]/th[1]/div/div[1]/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0',''),
                             dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[7]/th[1]/div/div[2]/text()')[0]
       .strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')]





NCT03093350["numberOfPatients"] = str(dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[4]/td/text()')[0]).strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')
    
NCT03093350["groupTitle"] = str(dom.xpath('//*[@id="tab-body"]/div[1]/div[2]/div[2]/table[2]/tbody/tr[1]/th/text()')[0]).strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')

NCT03093350["groupDescription"] = str(dom.xpath('//*[@id="EXPAND-armGroupDescriptionRow-baseline"]/td[2]//p/text()'))

NCT03093350["ageOfPatients"] = str(dom.xpath('//*[@id="tab-body"]/div[1]/div[3]/div[2]/table/tbody/tr[9]/td[2]/span/text()')[0]).strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')

NCT03093350["PFS"] = str(dom.xpath('//*[@id="EXPAND-outcome-data-2"]/table/tbody/tr[6]/td/div[1]/text()')[0]).strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')    

NCT03093350["PFS_unit"] = str(dom.xpath('//*[@id="EXPAND-outcome-data-2"]/table/tbody/tr[5]/th/div/text()')[0]).strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','') 

NCT03093350["OAS"] = str(dom.xpath('//*[@id="EXPAND-outcome-data-3"]/table/tbody/tr[6]/td/div[1]/text()')[0]).strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','')    

NCT03093350["OAS_unit"] = str(dom.xpath('//*[@id="EXPAND-outcome-data-3"]/table/tbody/tr[5]/th/div/text()')[0]).strip('\n').strip('\r').strip().replace('\n','').replace('\xa0','') 

kg_NCT03093350 = Graph()
kg_NCT03093350.bind('Clinical_Trials', CT)
kg_NCT03093350.bind('schema',SCHEMA)
kg_NCT03093350.add((ct_uri, CT.hasID, Literal('NCT03093350')))
            
ct_uri = URIRef('https://clinicaltrials.gov/ct2/show/NCT03093350')
kg_NCT03093350.add((ct_uri, RDF.type, SCHEMA.MedicalTrial))
title = str(NCT03093350["groupTitle"].replace(' ','_'))
kg_NCT03093350.add((ct_uri, CT.hasGroup, CT[title]))
kg_NCT03093350.add((CT[title], CT.hasDescription, Literal(NCT03093350["groupDescription"])))

kg_NCT03093350.add((ct_uri, CT.hasNumberofPatients, Literal(NCT03093350["numberOfPatients"])))
kg_NCT03093350.add((ct_uri, CT.hasMedianAge, Literal(NCT03093350["ageOfPatients"])))
kg_NCT03093350.add((ct_uri, CT.hasPFS, CT['NCT03093350_PFS']))
kg_NCT03093350.add((CT['NCT03093350_PFS'], CT.hasUnit, Literal(NCT03093350["PFS_unit"])))
kg_NCT03093350.add((CT['NCT03093350_PFS'], SCHEMA.number, Literal(NCT03093350["PFS"])))
kg_NCT03093350.add((ct_uri, CT.hasOAS, CT['NCT03093350_OAS']))
kg_NCT03093350.add((CT['NCT03093350_OAS'], CT.hasUnit, Literal(NCT03093350["OAS_unit"])))
kg_NCT03093350.add((CT['NCT03093350_OAS'], SCHEMA.number, Literal(NCT03093350["OAS"])))  

for i in NCT03093350:
    kg_NCT03093350.add((ct_uri, CT.hasStudyType, Literal(NCT03093350['StudyType'])))
    if i == 'InclusionCriteria':
        for j in NCT03093350[i]:
            kg_NCT03093350.add((ct_uri, CT.hasInclusionCriteria, Literal(j)))
    if i == 'ExclusionCriteria':
        for j in NCT03093350[i]:
            kg_NCT03093350.add((ct_uri, CT.hasExclusionCriteria, Literal(j)))
    if i == 'Drugs':
        for j in NCT03093350[i]:
            k = URIRef(NCT03093350[i][j]['url'])
            kg_NCT03093350.add((ct_uri, CT.hasDrug, k))
            kg_NCT03093350.add((k, SCHEMA.name, Literal(j)))
            kg_NCT03093350.add((k, SCHEMA.description, Literal(NCT03093350[i][j]['description'])))
            
kg_NCT03093350.serialize('new_graphs/NCT03093350.ttl',format="turtle")

NCT03093350["groupDescription"] = str(dom.xpath('//*[@id="EXPAND-armGroupDescriptionRow-baseline"]/td[2]//p/text()'))

NCT03093350["groupDescription"]
kg = Graph()
kg.bind('Clinical_Trials', CT)
kg.bind('schema',SCHEMA)
ct_uri = URIRef('https://clinicaltrials.gov/ct2/show/NCT04199078')
kg.add((ct_uri, RDF.type, SCHEMA.MedicalTrial))
kg.add((ct_uri, CT.hasID, Literal('NCT04199078')))
for i in paloma2:
    kg.add((ct_uri, CT.hasStudyType, Literal(paloma2['StudyType'])))
    if i == 'InclusionCriteria':
        for j in paloma2[i]:
            kg.add((ct_uri, CT.hasInclusionCriteria, Literal(j)))
    if i == 'ExclusionCriteria':
        for j in paloma2[i]:
            kg.add((ct_uri, CT.hasExclusionCriteria, Literal(j)))
    if i == 'Drugs':
        for j in paloma2[i]:
            k = URIRef(paloma2[i][j]['url'])
            kg.add((ct_uri, CT.hasDrug, k))
            kg.add((k, SCHEMA.name, Literal(j)))
            kg.add((k, SCHEMA.description, Literal(paloma2[i][j]['description'])))
    
    PFS_field = 'NCT04199078_PFS'
    PFS = 'NCT04199078_PFS_treatmentGroup'
    PFS1 = 'NCT04199078_PFS_controlGroup'
    kg.add((ct_uri, CT.hasPFS,CT[PFS_field]))
    kg.add((CT[PFS_field], CT.hasPFS,CT[PFS]))
    kg.add((CT[PFS], CT.hasGroup, Literal("treatmentGroup")))
    kg.add((CT[PFS], CT.hasUnit, Literal("month")))
    kg.add((CT[PFS], SCHEMA.number, Literal(PFS)))
    kg.add((CT[PFS_field], CT.hasPFS,CT[PFS1]))
    kg.add((CT[PFS1], CT.hasGroup, Literal("controlGroup")))
    kg.add((CT[PFS1], CT.hasUnit, Literal("month")))
    kg.add((CT[PFS1], SCHEMA.number, Literal(PFS1)))
    
    treatment_field = 'NCT04199078_treamentDescription'
    treatment = "NCT04199078_treamentGroupDescription"
    treatment1 = "NCT04199078_controlGroupDescription"
    kg.add((ct_uri, CT.hasTreatmentPlan,CT[treatment_field]))
    kg.add((CT[treatment_field], CT.hasTreatment,CT[treatment]))
    kg.add((CT[treatment], CT.hasGroup, Literal("treatmentGroup")))
    kg.add((CT[treatment], CT.hasdescription, Literal(paloma2["treatment"][0])))
    kg.add((CT[treatment_field], CT.hasTreatment,CT[treatment1]))
    kg.add((CT[treatment1], CT.hasGroup, Literal("controlGroup")))
    kg.add((CT[treatment1], CT.hasdescription, Literal(paloma2["treatment"][1])))
    
    nofP_field = 'NCT04199078_numberofPatients'
    numberofPatients = 'NCT04199078_numberofPatients_treatmentGroup'
    numberofPatients1 = 'NCT04199078_numberofPatients_controlGroup'
    kg.add((ct_uri, CT.hasNumberofPatients,CT[nofP_field]))
    kg.add((CT[nofP_field], CT.hasNumberofPatients,CT[numberofPatients]))
    kg.add((CT[numberofPatients], CT.hasGroup, Literal("treatmentGroup")))
    kg.add((CT[numberofPatients], SCHEMA.number, Literal(paloma2["numberOfPatients"][0])))
    kg.add((CT[numberofPatients], CT.hasUnit, Literal("person")))
    kg.add((CT[nofP_field], CT.hasNumberofPatients,CT[numberofPatients1]))
    kg.add((CT[numberofPatients1], CT.hasGroup, Literal("controlGroup")))
    kg.add((CT[numberofPatients1], SCHEMA.number, Literal(paloma2["numberOfPatients"][1])))
    kg.add((CT[numberofPatients1], CT.hasUnit, Literal("person")))
    
    aofP_field = 'NCT04199078_ageOfPatients_treatmentGroup'
    ageOfPatients = 'NCT04199078_ageOfPatients_controlGroup'
    ageOfPatients1 = paloma2["ageOfPatients"][1]
    kg.add((ct_uri, CT.hasMedianAge,CT[aofP_field]))
    kg.add((CT[aofP_field], CT.hasMedianAge,CT[ageOfPatients]))
    kg.add((CT[ageOfPatients], CT.hasGroup, Literal("treatmentGroup")))
    kg.add((CT[ageOfPatients], SCHEMA.number, Literal(paloma2["ageOfPatients"][0])))
    kg.add((CT[ageOfPatients], CT.hasUnit, Literal("year")))
    kg.add((CT[aofP_field], CT.hasMedianAge,CT[ageOfPatients1]))
    kg.add((CT[ageOfPatients1], CT.hasGroup, Literal("controlGroup")))
    kg.add((CT[ageOfPatients1], SCHEMA.number, Literal(paloma2["ageOfPatients"][1])))
    kg.add((CT[ageOfPatients1], CT.hasUnit, Literal("year")))

#      'hazardRatio': '0.576',
#  'hasPercentageOfPatientsWithVisceralDisease': [48.2, 49.5],
#  'hasPercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy': [48, 49.1],
#  'hasPercentageOfPatientsWithNewlyDiagonosedAdavancedDisease': 37.2}
    PVD_field = 'NCT04199078_percentageofPatientWithVisceraDisease'
    PVD = 'NCT04199078_percentageofPatientWithVisceraDisease_treatmentGroup'
    PVD1 = 'NCT04199078_percentageofPatientWithVisceraDisease_controlGroup'
    kg.add((ct_uri, CT.hasPercentageOfPatientsWithVisceralDisease,CT[PVD_field]))
    kg.add((CT[PVD_field], CT.hasPercentageOfPatientsWithVisceralDisease,CT[PVD]))
    kg.add((CT[PVD], CT.hasGroup, Literal("treatmentGroup")))
    kg.add((CT[PVD], SCHEMA.number, Literal(paloma2["hasPercentageOfPatientsWithVisceralDisease"][0])))
    kg.add((CT[PVD], CT.hasUnit, Literal("percentage")))
    kg.add((CT[PVD_field], CT.hasPercentageOfPatientsWithVisceralDisease,CT[PVD1]))
    kg.add((CT[PVD1], CT.hasGroup, Literal("controlGroup")))
    kg.add((CT[PVD1], SCHEMA.number, Literal(paloma2["hasPercentageOfPatientsWithVisceralDisease"][1])))
    kg.add((CT[PVD1], CT.hasUnit, Literal("percentage")))
    
    PriorChemo_field = 'NCT04199078_percentageofPatientsHadReceivedPriorAdjuvantChemotherapy'
    PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy = 'NCT04199078_percentageofPatientsHadReceivedPriorAdjuvantChemotherapy_treatmentGroup'
    PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy1 = 'NCT04199078_percentageofPatientsHadReceivedPriorAdjuvantChemotherapy_controlGroup'
    kg.add((ct_uri, CT.hasPercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy,CT[PriorChemo_field]))
    kg.add((CT[PriorChemo_field], CT.hasPercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy,
            CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy]))
    kg.add((CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy], CT.hasGroup, Literal("treatmentGroup")))
    kg.add((CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy], SCHEMA.number, Literal(paloma2["hasPercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy"][0])))
    kg.add((CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy], CT.hasUnit, Literal("percentage")))
    kg.add((CT[PriorChemo_field], CT.hasPercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy,
            CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy1]))
    kg.add((CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy1], CT.hasGroup, Literal("controlGroup")))
    kg.add((CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy1], SCHEMA.number, Literal(paloma2["hasPercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy"][1])))
    kg.add((CT[PercentageOfPatientsHadReceivedPriorAdjuvantChemotherapy1], CT.hasUnit, Literal("percentage")))    
    
    hazardRatio = paloma2["hazardRatio"]
    kg.add((ct_uri, CT.hasHazardRatio, CT['NCT04199078_hazardRatio']))
    kg.add((CT['NCT04199078_hazardRatio'], SCHEMA.number, Literal(paloma2["hazardRatio"])))
    kg.add((CT['NCT04199078_hazardRatio'], CT.hasConfidenceInterval, Literal("(2 sided)95% 0.463 to 0.718")))
    
    PercentageOfPatientsWithNewlyDiagonosedAdavancedDisease = str(paloma2["hasPercentageOfPatientsWithNewlyDiagonosedAdavancedDisease"])
    kg.add((ct_uri, CT.hasPercentageOfPatientsWithNewlyDiagonosedAdavancedDisease, CT['NCT04199078_PercentageOfPatientsWithNewlyDiagonosedAdavancedDisease']))
    kg.add((CT['NCT04199078_PercentageOfPatientsWithNewlyDiagonosedAdavancedDisease'], SCHEMA.number, Literal(paloma2["hasPercentageOfPatientsWithNewlyDiagonosedAdavancedDisease"])))
    kg.add((CT['NCT04199078_PercentageOfPatientsWithNewlyDiagonosedAdavancedDisease'], CT.hasUnit, Literal("percentage")))
    
kg.serialize('paloma-2.ttl', format="turtle")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20770634404839275, 'fraction_numerical': 0.04330600409938509, 'mean_word_length': 5.36033386327504, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 39, 'https://': 67, 'lorem ipsum': 0, 'www.': 1, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '588262', 'license': 'Apache-2.0', 'n_tokens_mistral': 32336, 'n_tokens_neox': 30020, 'n_words': 2916, 'path': 'CTKG-7-28.ipynb', 'repo_name': 'KnowledgeCaptureAndDiscovery/clinincal_trials', 'size': '80012'}