<jupyter_start><jupyter_text>Let's try to query all the archiveTypes present on the wiki. This information is available through two properties:- ArchiveType present at the dataset level (not part of core)- ProxyArchiveType available at the variable level (part of core)This is the query from core:<jupyter_code>import json
import requests

url = "http://wiki.linked.earth/store/db/query"

query = """PREFIX core: <http://linked.earth/ontology#>
PREFIX wiki: <http://wiki.linked.earth/Special:URIResolver/>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
SELECT distinct ?a 
WHERE {
{
    ?w core:proxyArchiveType ?t.
    ?t rdfs:label ?a
}
}"""

response = requests.post(url, data = {'query': query})
res = json.loads(response.text)
print(response)
print("The following archive types are available on the wiki:")    
for item in res['results']['bindings']:
    print ("*" + item['a']['value'])<jupyter_output><Response [200]>
The following archive types are available on the wiki:
*Coral
*GlacierIce
*LakeSediment
*MarineSediment
*Speleothem<jupyter_text>Doesn't return any of the ProxyArchiveTypes.This is the response from wiki:<jupyter_code>import json
import requests

url = "http://wiki.linked.earth/store/db/query"

query = """PREFIX core: <http://linked.earth/ontology#>
PREFIX wiki: <http://wiki.linked.earth/Special:URIResolver/>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
SELECT distinct ?a 
WHERE {
{
    ?dataset wiki:Property-3AArchiveType ?a.
}
}"""

response = requests.post(url, data = {'query': query})
res = json.loads(response.text)
print(response)
print("The following archive types are available on the wiki:")    
for item in res['results']['bindings']:
    print ("*" + item['a']['value'])<jupyter_output><Response [200]>
The following archive types are available on the wiki:
*tree
*marine sediment
*speleothem
*lake sediment
*glacier ice
*borehole
*documents
*hybrid
*coral
*bivalve
*sclerosponge<jupyter_text>Trying to go around the core issue is not easy because I don't know how to fix the URL interpretation of ':', '_', '(',')' Trying to query through the wiki prefix but not sure how to deal with special characters in URL. See below:<jupyter_code># DataSetName = ["Aus-StewartIsland.D'Arrigo.1996", "Arc-BigRoundLake.Thomas.2009"]
DataSetName = ["ME0005A-43JC.Benway.2006"]
import json
import requests
import pandas as pd

url = "http://wiki.linked.earth/store/db/query"

query = """PREFIX core: <http://linked.earth/ontology#>
PREFIX wiki: <http://wiki.linked.earth/Special:URIResolver/>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
SELECT ?dataset ?a
WHERE {"""

query+= "VALUES ?b {"
for item in DataSetName:
    query +="\""+item+"\" "
query+= "}\n"

query+= "?dataset wiki:Property-3AName%5F%28L%29 ?b. \n"
query+= ' ?dataset wiki:Property-3AArchiveType ?a \n}'

response = requests.post(url, data = {'query': query})
res = json.loads(response.text)

print(response,res)
#archiveType = []
#for item in res['results']['bindings']:
#    archiveType.append(item['a']['value'])

#results = pd.DataFrame({'dataset name': DataSetName,'archive type': archiveType})
#results<jupyter_output><Response [200]> {'head': {'vars': ['dataset', 'a']}, 'results': {'bindings': []}}<jupyter_text>Get all the LiPD filenames from the wiki<jupyter_code>import json
import requests

url = "http://wiki.linked.earth/store/db/query"

query = """PREFIX core: <http://linked.earth/ontology#>
PREFIX wiki: <http://wiki.linked.earth/Special:URIResolver/>
SELECT ?ds ?name
WHERE {
  ?ds a core:Dataset .
  ?ds core:name ?name
}"""

response = requests.post(url, data = {'query': query})
res = json.loads(response.text)

files_in_wiki = set()
for item in res['results']['bindings']:
    files_in_wiki.add(item['name']['value'])
files_in_wiki<jupyter_output><empty_output><jupyter_text>For a filename , get the archive Type<jupyter_code>DataSetName = ["Aus-StewartIsland.D'Arrigo.1996", "Arc-BigRoundLake.Thomas.2009"]
# DataSetName = ["Afr-LakeMalawi.Powers.2011"]<jupyter_output><empty_output><jupyter_text>These datasets have an archiveType.  http://wiki.linked.earth/Aus-StewartIsland.D%27Arrigo.1996  http://wiki.linked.earth/Arc-BigRoundLake.Thomas.2009<jupyter_code>import json
import requests
import pandas as pd

url = "http://wiki.linked.earth/store/db/query"

query = """PREFIX core: <http://linked.earth/ontology#>
PREFIX wiki: <http://wiki.linked.earth/Special:URIResolver/>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
SELECT ?dataset ?a ?pot
WHERE {"""

query+= "VALUES ?b {"
for item in DataSetName:
    query +="\""+item+"\" "
query+= "}\n"

query+= "?dataset core:name ?b. \n"
query+= ' ?dataset wiki:Property-3AArchiveType ?a. \n}'
# query+= ' ?dataset core:includesPaleoData ?pd. \n}'
# query+= ' ?pd core:foundInMeasurementTable ?table. \n}'
# query+= ' ?table core:includesVariable ?var . \n}'
# query+= ' ?var core:proxyObservationType ?pot \n}'

response = requests.post(url, data = {'query': query})
res = json.loads(response.text)

#archiveType = []
#for item in res['results']['bindings']:
#    archiveType.append(item['a']['value'])

#results = pd.DataFrame({'dataset name': DataSetName,'archive type': archiveType})
#results

print(response, res)<jupyter_output><Response [200]> {'head': {'vars': ['dataset', 'a', 'pot']}, 'results': {'bindings': [{'dataset': {'type': 'uri', 'value': 'http://wiki.linked.earth/Special:URIResolver/Aus-2DStewartIsland.D-27Arrigo.1996'}, 'a': {'type': 'literal', 'value': 'tree'}}, {'dataset': {'type': 'uri', 'value': 'http://wiki.linked.earth/Special:URIResolver/Arc-2DBigRoundLake.Thomas.2009'}, 'a': {'type': 'literal', 'value': 'lake sediment'}}]}}<jupyter_text>To see how many lipd files have paleodata from all the files in the wiki<jupyter_code>import os
import sys
import json
import requests
import urllib

# datasetname = "Afr-LakeMalawi.Powers.2011"
datasetname = "Arc-BigRoundLake.Thomas.2009"

endpoint = "http://wiki.linked.earth/store/db/query" # Query Metadata
wikiapi = "http://wiki.linked.earth/wiki/api.php" # Fetch file data

query = """PREFIX core: <http://linked.earth/ontology#>
PREFIX wiki: <http://wiki.linked.earth/Special:URIResolver/>
SELECT ?name ?data ?pot
WHERE {
  ?ds a core:Dataset .
  ?ds core:name ?name .
  ?ds core:includesPaleoData ?data .
  ?data core:foundInMeasurementTable ?table .
  ?table core:includesVariable ?var .
}"""
# ?data core:foundInMeasurementTable ?table .
#   ?table core:includesVariable ?var .
#   ?var core:proxyObservationType ?pot.

# ?table core:hasFileName ?filename

# ?ds core:name '""" + datasetname + """' .
response = requests.post(endpoint, data = {'query': query})
res = json.loads(response.text)
# print(response, res)

files_with_paleo = set()
for item in res['results']['bindings']:
    files_with_paleo.add(item['name']['value'])
len(files_with_paleo)
file_list = list(files_with_paleo.intersection(files_in_wiki))
print(file_list)
len(file_list)
import pandas as pd
master_table = pd.DataFrame(columns = ['dataset name', 'archiveType', 'proxyObservationType', 'units', 'proxySystem'])<jupyter_output><empty_output><jupyter_text>Get all the proxyObsTypes for files that have paleodata<jupyter_code>import os
import sys
import json
import requests
import urllib

no_content_files = []
no_paleodata = []
# datasetname = "Afr-LakeMalawi.Powers.2011"
# datasetname = "Afr-LakeTanganyika.Tierney.2010"
# file_list=['NAm-SnowBowlSanFranciscoPeak.Briffa.2002', "NAm-SewardComposite.D'Arrigo.2006", 'Ocn-AbrahamReef.Druffel.1999']
# file_list = ['Arc-BigRoundLake.Thomas.2009']

endpoint = "http://wiki.linked.earth/store/db/query" # Query Metadata
wikiapi = "http://wiki.linked.earth/wiki/api.php" # Fetch file data

for datasetname in file_list:
    query = """PREFIX core: <http://linked.earth/ontology#>
    PREFIX wiki: <http://wiki.linked.earth/Special:URIResolver/>
    SELECT ?name ?a ?potName ?uni ?psys
    WHERE {
    
       # binding defaults
      bind("NA" as ?default_units)
      bind("NA" as ?default_psys)
      
      ?ds a core:Dataset .
      ?ds core:name ?name .
      ?ds core:name '""" + datasetname + """' .
      ?ds wiki:Property-3AArchiveType ?a.
      ?ds core:includesPaleoData ?data .
      ?data core:foundInMeasurementTable ?table .
      ?table core:includesVariable ?var .
      ?var core:proxyObservationType ?pot .
      ?var core:name ?potName .
      
      OPTIONAL {
          ?var core:hasUnits ?wiki_uni .
          ?var core:hasProxySystem ?wiki_psys .
      }
      
      # either take ?wiki_uni/wiki_psys value or default and bind it as the result ?uni/psys
      bind(coalesce(?wiki_uni, ?default_units) as ?uni)
      bind(coalesce(?wiki_psys, ?default_psys) as ?psys)
      
    }"""

    try:
        response = requests.post(endpoint, data = {'query': query})
        res = json.loads(response.text)
        print(response, res)
        print('$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$')
        
        if not res['results']['bindings']:
            no_content_files.append(datasetname)
        for item in res['results']['bindings']:
            df = pd.DataFrame({'dataset name':item['name']['value'], 'archiveType': item['a']['value'], 'proxyObservationType':item['potName']['value'], 'units':item['uni']['value'], 'proxySystem':item['psys']['value']}, index=[0])
            master_table = master_table.append(df, ignore_index = True)
    except:
        no_paleodata.append(datasetname)
        print('Dataset does not have Paleodata :(')


master_table.to_csv(r'D:\\annotating_paleoclimate_data\\proxyObsTypeWiki.csv', sep = ',', encoding = 'utf-8',index = False)
print(len(no_content_files))
no_content_files
import os
import sys
import json
import requests
import urllib

# datasetname = "Afr-LakeMalawi.Powers.2011"
datasetname = "Arc-BigRoundLake.Thomas.2009"

endpoint = "http://wiki.linked.earth/store/db/query" # Query Metadata
wikiapi = "http://wiki.linked.earth/wiki/api.php" # Fetch file data

query = """PREFIX core: <http://linked.earth/ontology#>
PREFIX wiki: <http://wiki.linked.earth/Special:URIResolver/>
SELECT ?name ?data ?table ?pot
WHERE {
  ?ds a core:Dataset .
  ?ds core:name ?name .
  ?ds core:name '""" + datasetname + """' .
  ?ds core:includesPaleoData ?data .
  ?data core:foundInMeasurementTable ?table .
  
}"""
# ?data core:foundInMeasurementTable ?table .
#   ?table core:includesVariable ?var .
#   ?var core:proxyObservationType ?pot.

response = requests.post(endpoint, data = {'query': query})
res = json.loads(response.text)
print(response, res)<jupyter_output><Response [200]> {'head': {'vars': ['name', 'data', 'table', 'pot']}, 'results': {'bindings': [{'name': {'type': 'literal', 'value': 'Arc-BigRoundLake.Thomas.2009'}, 'data': {'type': 'uri', 'value': 'http://wiki.linked.earth/Special:URIResolver/Arc-2DBigRoundLake.Thomas.2009.PaleoData1'}, 'table': {'type': 'uri', 'value': 'http://wiki.linked.earth/Special:URIResolver/Arc-2DBigRoundLake.Thomas.2009.paleo1measurement1'}}, {'name': {'type': 'literal', 'value': 'Arc-BigRoundLake.Thomas.2009'}, 'data': {'type': 'uri', 'value': 'http://wiki.linked.earth/Special:URIResolver/Arc-2DBigRoundLake.Thomas.2009.PaleoData2'}, 'table': {'type': 'uri', 'value': 'http://wiki.linked.earth/Special:URIResolver/Arc-2DBigRoundLake.Thomas.2009.paleo2measurement1'}}]}}
{'fraction_non_alphanumeric': 0.17512130569033965, 'fraction_numerical': 0.015968239964711074, 'mean_word_length': 4.985216473072862, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '559278', 'license': 'Apache-2.0', 'n_tokens_mistral': 4091, 'n_tokens_neox': 3866, 'n_words': 862, 'path': 'background_material/SPARQL_queries/querying for archive type.ipynb', 'repo_name': 'paleopresto/recommender', 'size': '11335'}