<jupyter_start><jupyter_text>import packages<jupyter_code>import pandas as pd
import sys
sys.path.append("/Users/tong/Desktop/ISI/SPARQL/")
import find_identity
import numpy as np<jupyter_output><empty_output><jupyter_text>read data from .csv file, transfer them to DataFrame<jupyter_code>data = pd.read_csv('./learningData.csv')
df = pd.DataFrame(data)
df.head()<jupyter_output><empty_output><jupyter_text>Input FIPS and State(2&3 column in file), find "PNode", output their "QNode"<jupyter_code>for column in df.columns[2:3]:
    curData = [str(x) for x in list(df[column])]
    for i in range(len(curData)):
        if len(curData[i]) == 4:
            curData[i] = '0' + curData[i]
    f = find_identity.find_identity()
    for idx,res in enumerate(f.get_identifier_3(curData)):
        top1_dict=res[1]
        new_col = []
        for i in range(len(curData)):
            if curData[i] in top1_dict:
                new_col.append(top1_dict[curData[i]])
            else:
                new_col.append('')
        col_name = column+'_wikidata'
        df[col_name] = new_col
        break
df.head()<jupyter_output><empty_output><jupyter_text>Save new DataFrame into .csv file<jupyter_code>df.to_csv('./learningData_wikify.csv')
import json, requests
def call_redis(qnodes):
    payload = json.dumps({"ids": qnodes})
    search_headers = {'Accept': 'application/json', 'Content-Type': 'application/json'}
    r = requests.post("http://minds03.isi.edu:4444/get_identifiers",
        data=payload,headers=search_headers)
    if r:
        val = r.json()
        return val
    else:
        return {}
f.call_redis(['25145adfadf'])
print("四".isdigit())
print("四".isnumeric())
data = pd.read_csv('./SPARQL/Book1.csv')
df = pd.DataFrame(data)
df.head()
f = find_identity.find_identity()
input_list = [x + ' County' for x in list(df['county'])]
f.get_identifier_3(input_list)
list(df['county'])
f.call_redis(['35005'])
input_list[:10]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1486005089058524, 'fraction_numerical': 0.015776081424936386, 'mean_word_length': 3.9772151898734176, 'pattern_counts': {'":': 1, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '345727', 'license': 'MIT', 'n_tokens_mistral': 726, 'n_tokens_neox': 712, 'n_words': 131, 'path': 'wikifier/IdentifierWikifier.ipynb', 'repo_name': 'usc-isi-i2/datamart-user-end', 'size': '1965'}