<jupyter_start><jupyter_text>Postprocessing<jupyter_code>G=json.loads(file('../json/pop1.json','r').read())
error=json.loads(file('../json/pop1e.json','r').read())
print len(G),len(error)
error2=[]
import unicodedata
def strip_accents(s):
   return ''.join(c for c in unicodedata.normalize('NFD', s)
                  if unicodedata.category(c) != 'Mn')
def remove_accents(input_str):
    nfkd_form = unicodedata.normalize('NFKD', input_str)
    only_ascii = nfkd_form.encode('ASCII', 'ignore')
    return only_ascii
for c in error:
    if c not in G.keys()+error2:
        ys={"pop":0,"nearby":{}}
        q=remove_accents(strip_accents('population of '+c.split('/')[0]+', '+unicities[c]))
        res = client.query(q)
        if 'pod' in res:
            for i in range(len(res['pod'])):
                if res['pod'][i]['@title']=="Result":
                    x=res['pod'][i]['subpod']['plaintext']
                    if 'available' not in x:
                        popul=x[:x.find('people')-1]
                        if 'mill' in popul:
                            popul=popul[:popul.find('mill')-1]
                            if '|' in popul:popul=popul.split('|')[1].strip()
                        ys['pop']=int(float(popul)*1000000.0)
                if res['pod'][i]['@title']=="Nearby cities":
                    x=res['pod'][i]['subpod']['plaintext'].split('\n')
                    if 'available' not in x:
                        for y in x[:-1]:
                            people=y[y.rfind('|')+2:y.find('people')-1]
                            if 'mill' in people:
                                people=float(people[:people.find('mill')-1])*1000000.0
                            km=float(y[y.find('|')+2:y.find(' km ')])
                            ys['nearby'][y.split('|')[0].split(',')[0].strip()]={"km":km,"people":int(people)}
                        G[c]=ys
                        print 'success',c
        else: 
            print 'error',c
            error2.append(c)
file("../json/pop1b.json",'w').write(json.dumps(G))
file("../json/pop1eb.json",'w').write(json.dumps(error2))
print len(G),len(error),len(error2)<jupyter_output>792 121 100<jupyter_text>Re-reparse<jupyter_code>G=json.loads(file('../json/pop1b.json','r').read())
error=json.loads(file('../json/pop1eb.json','r').read())
error3=[]
error4=[]
uk=unicities.keys()
for h in range(900):
    c=uk[h]
    if c not in G:
        error3.append(c)
print len(error3)
for c in error3:
    if c not in G.keys()+error4:
        ys={"pop":0,"nearby":{}}
        q=remove_accents(strip_accents('population of '+c.split('/')[0]+', '+unicities[c]))
        res = client.query(q)
        good=True
        if 'pod' in res:
            for i in range(len(res['pod'])):
                try:
                    if res['pod'][i]['@title']=="Result":
                        x=res['pod'][i]['subpod']['plaintext']
                        if 'available' not in x:
                            popul=x[:x.find('people')-1]
                            if 'mill' in popul:
                                popul=popul[:popul.find('mill')-1]
                                if '|' in popul:popul=popul.split('|')[1].strip()
                            ys['pop']=int(float(popul)*1000000.0)
                    if res['pod'][i]['@title']=="Nearby cities":
                        x=res['pod'][i]['subpod']['plaintext'].split('\n')
                        if 'available' not in x:
                            for y in x[:-1]:
                                people=y[y.rfind('|')+2:y.find('people')-1]
                                if 'mill' in people:
                                    people=float(people[:people.find('mill')-1])*1000000.0
                                km=float(y[y.find('|')+2:y.find(' km ')])
                                ys['nearby'][y.split('|')[0].split(',')[0].strip()]={"km":km,"people":int(people)}
                            G[c]=ys
                            print 'success',c
                            good=False
                except: pass
        if good: 
            print 'error',c
            error4.append(c)
file("../json/pop1c.json",'w').write(json.dumps(G))
file("../json/pop1ec.json",'w').write(json.dumps(error4))
print len(G),len(error4)<jupyter_output>792 108
{'fraction_non_alphanumeric': 0.1927654609101517, 'fraction_numerical': 0.022403733955659276, 'mean_word_length': 1.6837820914214152, 'pattern_counts': {'":': 12, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '169113', 'license': 'MIT', 'n_tokens_mistral': 1464, 'n_tokens_neox': 1393, 'n_words': 205, 'path': 'code/population1.ipynb', 'repo_name': 'csaladenes/aviation', 'size': '4285'}