<jupyter_start><jupyter_text>Article web page processing<jupyter_code>import html2text
import json
import requests
import numpy as np
from bs4 import BeautifulSoup
def text_from_html(url: str): #data_folder_html_file_id):
    """
    Given url of the webpage returns the readable text from the page
    """
    #road_trip = open("../data/" + str(data_folder_html_file_id) + ".html", "r").read()
    road_trip = requests.get(url).text
    soup = BeautifulSoup(road_trip)
    links = []
    for link in soup.findAll('a'):
        http_link = link.get('href')
        if "nationalgeographic" in http_link:
            continue
        if not http_link.startswith("http"):
            continue
        if len(http_link) > 5:
            links.append(http_link)
    
    text_maker = html2text.HTML2Text()
    text_maker.ignore_links = True
    text_maker.ignore_images = True
    road_trip_processed = text_maker.handle(road_trip)
    
    start = "ShareTweetEmail"
    index = road_trip_processed.find(start)
    road_trip_processed = road_trip_processed[index + len(start) + 1:]
    
    end = road_trip_processed.find("ShareTweetEmail")
    road_trip_processed = road_trip_processed[:end]
    road_trip_processed = road_trip_processed.replace("\n\n", ". ")
    for i in ["\n", "[", "]", "*", "#", "_", "\\'", '\\']:
        road_trip_processed = road_trip_processed.replace(i, " ")
    road_trip_processed = road_trip_processed.replace('>', ", ")
    for i in range(4): # 4 seems enough
        road_trip_processed = road_trip_processed.replace("  ", " ")
    road_trip_processed = road_trip_processed.replace("..", ".")
               
    return road_trip_processed.strip(), links

def get_text_and_places() -> dict:
    with open('/home/geoner/data/articles.json') as json_file:
        data = json.load(json_file)
    for i in data:
        text, _ = text_from_html(data[i]["article_url"])
        data[i]["text"] = text      
    return data
%time data = get_text_and_places()<jupyter_output>CPU times: user 934 ms, sys: 33.6 ms, total: 968 ms
Wall time: 8.76 s<jupyter_text>Extractor<jupyter_code>import spacy
nlp = spacy.load('en_core_web_trf')
def get_ne(txt: str) -> list:
    ne_types = ["GPE", "FAC", "LOC", "ORG"]
    ne = nlp(txt).ents
    target_ne = [[str(ne[i]).strip(), ne[i]] for i in range(len(ne)) if ne[i].label_ in ne_types]
    for i in range(len(target_ne)):
        if target_ne[i][0].startswith("the"):
            target_ne[i][0] = target_ne[i][0][4:]
        if target_ne[i][0].endswith(','):
            target_ne[i][0] = target_ne[i][0][:-1]
        target_ne[i][0] = target_ne[i][0].strip()
        if target_ne[i][0].endswith('\'s'):
            target_ne[i][0] = target_ne[i][0][:-2]
        if target_ne[i][0].endswith("’s"):
            target_ne[i][0] = target_ne[i][0][:-2]    
        target_ne[i][0] = target_ne[i][0].strip()
    places = list(set(i[0] for i in target_ne)) # removing duplicates as well
    places_and_poi_processed = []
    # case: route 66 museum -> oklahoma route 66 museum
    for i in places:
        if i.startswith("National Geographic"):
            continue
        if i.startswith("Peace of Mind"):
            continue
        if i.endswith(" and"):
            continue
        places_and_poi_processed.append(i)
    places_and_poi = [i for i in target_ne if i[0] in places_and_poi_processed]
    places_and_poi.sort(key=lambda x: x[1].start_char)
    ne_index = {}
    for i in range(len(places_and_poi)):
        if ne_index.get(places_and_poi[i][0]):
            ne_index[places_and_poi[i][0]].append(i)
        else:
            ne_index[places_and_poi[i][0]] = [i]
    return places_and_poi_processed, places_and_poi, ne_index<jupyter_output><empty_output><jupyter_text>Gazetteer<jupyter_code>import difflib
def str_similarity_percent(a: str, b: str) -> float:
    return round(difflib.SequenceMatcher(lambda x: x == " ", a, b).ratio(), 3)
# 20'000 credits daily limit per application (identified by the parameter 'username'),
# the hourly limit is 1000 credits. A credit is a web service request hit for most services.
# An exception is thrown when the limit is exceeded. 
geonames = {
        "exact_match":"http://secure.geonames.org/searchJSON?maxRows=40&username=kwh44&name_equals=",
    "general_match": "http://secure.geonames.org/searchJSON?maxRows=30&username=kwh44&name="
}
# https://www.gisgraphy.com/documentation/user-guide.php#fulltextservice
gisgraphy = {
    "exact_match": "https://services.gisgraphy.com/fulltext/fulltextsearch?&format=json&allwordsrequired=true&q=",
    "general_match": "https://services.gisgraphy.com/fulltext/fulltextsearch?&format=json&q="
}
resource_name = ""
dbpedia = {
    "resource_name_match": "https://lookup.dbpedia.org/api/search?&typeName=place&format=JSON_FULL&maxResults=3&query=",
    "resource_rdf_get": f"https://dbpedia.org/data/{resource_name}.json"
}<jupyter_output><empty_output><jupyter_text>Existence Detection<jupyter_code>import reverse_geocoder as rg
import unidecode
def get_state_by_lat_lng(lat, lng):
    return rg.search((lat,lng))[0]
class Place:
    
    def __init__(self, place_name, country, state, place_type, lat, long, link=None, population=0):
        self.place_name = place_name
        self.country = country
        self.state = state
        self.place_type = place_type
        self.lat = float(lat)
        self.long = float(long)
        self.link = link
        self.population = population
        
    def __repr__(self):
        return f"Place name: {self.place_name}; State: {self.state}, \
Country: {self.country}, Place Type: {self.place_type}, Lat: {self.lat}, Long: {self.long}"
    
    def __eq__(self, other):
        return self.place_name == other.place_name and self.country == other.country and \
        self.state == other.state
# can be made faster by parallelizing
def existence_check(extracted_places: list) -> tuple:
    geonames_search = dict()
    gisgraphy_search = dict()
    dbpedia_search = dict()
    for i in extracted_places:
        print(i)
        # try dbpedia, maybe there is a wikipedia page for the place extracted
        #print("DBpedia search")
        # dbpedia resource search
        dbpedia_lookup = requests.get(dbpedia["resource_name_match"] + i).json()
        for result in dbpedia_lookup["docs"]:
            result_name = result["label"][0]["value"]
            redirect_result_name = []
            if result.get("redirectlabel"):
                redirect_result_name = [k["value"] for k in result["redirectlabel"]]
            relevance = max(str_similarity_percent(unidecode.unidecode(r), i) for r in redirect_result_name + [result_name])
            print("Result relevance is ", relevance)
            if relevance >= 0.65:
                print("Wikipedia page found")
                # do resourse rdf json get
                rdf_json_url = result["resource"][0]["value"].replace("resource", "data") + ".json"
                print(rdf_json_url)
                try:
                    resource_rdf = requests.get(rdf_json_url, timeout=5).json()
                except:
                    print("dbpedia timeout")
                    continue
                if dbpedia_search.get(i):
                    dbpedia_search[i].append([result["resource"][0]["value"], resource_rdf])
                else:
                    dbpedia_search[i] = [[result["resource"][0]["value"], resource_rdf]]
        exact_match = False
        # geonames exact place match search
        geoname_exact_lookup = requests.get(geonames["exact_match"] + i).json()
        # print(geoname_exact_lookup)
        results_num = geoname_exact_lookup["totalResultsCount"]
        if results_num >= 1:
            geonames_search[i] = geoname_exact_lookup
            print("Geonames exact search found match")
            exact_match = True
        # gisgraphy exact place match search
        gisgraphy_exact_lookup = requests.get(gisgraphy["exact_match"] + i).json()
        # print(gisgraphy_exact_lookup)
        results_num = gisgraphy_exact_lookup["response"]["numFound"]
        if results_num >= 1:
            # gisgaphy is not that good with exact match, might find lots of places that constain the query string,
            # but those places would add more noise for region localization module
            if results_num < 80:
                gisgraphy_search[i] = gisgraphy_exact_lookup
                print("Gisgraphy exact search found match")
                exact_match = True
        if exact_match:
            continue
        else:
            print("exact place name search failed")        
        # exact place name search failed
        # geonames partial place match search
        geoname_general_lookup = requests.get(geonames["general_match"] + i).json()
        results_num = geoname_general_lookup["totalResultsCount"]
        if results_num >= 1:
            geonames_search[i] = geoname_general_lookup
            print("Geonames partial place match found")
            continue
        # gisgraphy partial place match search
        gisgraphy_general_lookup = requests.get(gisgraphy["general_match"] + i).json()
        results_num = gisgraphy_general_lookup["response"]["numFound"]
        if results_num >= 1:
            if results_num < 80:
                gisgraphy_search[i] = gisgraphy_general_lookup
                print("Geonames partial place match found")
                continue
    return geonames_search, gisgraphy_search, dbpedia_search
def unify_lookup_results(lookup_response):
    geonames_results, gisgraphy_results, dbpedia_rdf = lookup_response
    unified_lookup_results = []
    for key, value in geonames_results.items():
        results = value['geonames']
        for res in results:
            if str_similarity_percent(key, res["toponymName"]) < 0.65:
                continue
            # skip results that don't have required information
            try:
                lat = res["lat"]
                long = res["lng"]
                place_name = key # res["toponymName"]
                country = res["countryCode"]
                place_type = res["fcodeName"]
                population = res.get("population", 0)
                state = res["adminName1"]
                unified_lookup_results.append(Place(place_name, country, state, place_type, lat, long, population=population))
            except:
                continue
    for key, value in gisgraphy_results.items():
        num = len(value["response"]["docs"])
        for res in value["response"]["docs"]:
            if str_similarity_percent(key, res['name']) < 0.75:
                continue
            try:
                lat = res["lat"]
                long = res["lng"]
                place_name = key
                country = res["country_code"]
                place_type = res["placetype"]
                population = res.get("population", 0)
                state = get_state_by_lat_lng(lat, long)["admin1"]
                unified_lookup_results.append(Place(place_name, country, state, place_type, lat, long, population=population))
            except:
                continue
    for key, value in dbpedia_rdf.items():
        for result in value:
            try:
                rdf = result[1][result[0]]
                long = rdf["http://www.w3.org/2003/01/geo/wgs84_pos#long"][0]["value"]
                lat = rdf["http://www.w3.org/2003/01/geo/wgs84_pos#lat"][0]["value"]
                place_type = "Attraction"
                place_name = key
                country_admin1 = get_state_by_lat_lng(lat, long)
                state = country_admin1["admin1"]
                country = country_admin1["cc"]
                unified_lookup_results.append(Place(place_name, country, state, place_type, lat, long))
            except Exception as e:
                continue
    return unified_lookup_results<jupyter_output><empty_output><jupyter_text>Region localization and noise removal<jupyter_code>from geopy import distance

def ll_dist(a, b):
    return distance.geodesic(a, b).km
def leave_one_country(geo_points):
    places = geo_points
    countries = dict()
    for i in places:
        if countries.get(i.country):
            countries[i.country] +=1
        else:
            countries[i.country] = 1
    travel_country = max(countries, key=lambda x: countries[x])
    travel_country_places = list(filter(lambda i: i.country == travel_country, places))
    place_name_geocoding = dict()
    for place in travel_country_places:
        if place_name_geocoding.get(place.place_name):
            place_name_geocoding[place.place_name].append(place)
        else:
            place_name_geocoding[place.place_name] = [place]
    states = [i.state for i in travel_country_places if i.place_type == "first-order administrative division"]
    return place_name_geocoding, states
def remove_duplicates_and_leave_states(place_name_geocoding, states):
    for i in place_name_geocoding:
        possible_geolocation_num = len(place_name_geocoding[i]) 
        if possible_geolocation_num <= 1:
            continue
        filtered_geolocations = []
        # state names are treated as first-order administrative division
        for j in place_name_geocoding[i]:
            if j.place_type == "first-order administrative division":
                filtered_geolocations.append(j)
                break
        if filtered_geolocations:
            place_name_geocoding[i] = filtered_geolocations
            continue
        filtered_geolocations = []    
        # remove duplicate geolocations that come from different
        # gazetteers and differ slightly in coordinates
        for j in place_name_geocoding[i]:
#             if j.state not in states:
#                 continue
            duplicate = False
            for q in place_name_geocoding[i]:
                if j.state == q.state and abs(j.lat - q.lat) < 1.0 and abs(j.long - q.long) < 1.0:
                    duplicate = True
            if not duplicate or j not in filtered_geolocations:
                filtered_geolocations.append(j)
        place_name_geocoding[i] = filtered_geolocations
        # remove geolocations that are located on state borders,
        # that point to the same place, but in on gazetteer it on opposing sides of the border
        filtered_geolocations = []    
        for j in place_name_geocoding[i]:
            duplicate = False
            for q in filtered_geolocations:
                if j != q and abs(j.lat - q.lat) < 1.0 and abs(j.long - q.long) < 1.0:
                    duplicate = True
            if not duplicate:
                filtered_geolocations.append(j)
        place_name_geocoding[i] = filtered_geolocations
    return place_name_geocoding
def located_based_on_neighbours_v2(place_name_geocoding, ne_index, places_and_poi, order=0, gap=1):
    for i in place_name_geocoding:    
        # for places that have two or more possible geolocations
        # look at left and right target token in the text to 
        # decide which geolocation to choose
        neighbours = []
        if len(place_name_geocoding[i]) > 1:
            try:
                first_mention = ne_index[i][order]
            except:
                pass
            try:
                print(i)
                right_neighbour = places_and_poi[first_mention + gap]
                right_neighbour_name = right_neighbour[0]
                print("Right neighbour ", right_neighbour_name)
                if len(place_name_geocoding[right_neighbour_name]) == 1:
                    neighbours.append(place_name_geocoding[right_neighbour_name][0])
            except:
                pass
            try:
                left_neighbour = places_and_poi[first_mention - gap]
                left_neighbour_name = left_neighbour[0]
                print("left neighbour ", left_neighbour_name)
                if len(place_name_geocoding[left_neighbour_name]) == 1:
                    neighbours.append(place_name_geocoding[left_neighbour_name][0])
            except:
                pass
        if neighbours:
            filtered_geolocations = []
            closest_neighbour = None
            closest_dist = None
            closest_possible_geolocation = None
            for n in range(len(neighbours)):
                cord1 = (neighbours[n].lat, neighbours[n].long)
                for b in place_name_geocoding[i]:
                    cord2 = (b.lat, b.long)
                    d = ll_dist(cord1, cord2)
                    if closest_dist is None or closest_dist > d:
                        closest_dist = d
                        closest_neighbour = neighbours[n]
                        closest_possible_geolocation = b
            print("Closest neighbour ", closest_neighbour.place_name)
            print("Closest distance ", closest_dist)
            if closest_dist <= 100:
                # in km
                place_name_geocoding[i] = [closest_possible_geolocation]
    return place_name_geocoding
def located_based_on_neighbours(place_name_geocoding, ne_index, places_and_poi, order=0, gap=1):
    for i in place_name_geocoding:    
        # for places that have two or more possible geolocations
        # look at left and right target token in the text to 
        # decide which geolocation to choose
        neighbours_state = []
        if len(place_name_geocoding[i]) > 1:
            try:
                first_mention = ne_index[i][order]
            except:
                continue
            try:
                right_neighbour = places_and_poi[first_mention + gap]
                dist = abs(right_neighbour[1].start_char - places_and_poi[first_mention][1].start_char)
                if dist > 300:
                    raise ValueError("neighbour too far away")
                right_neighbour_name = right_neighbour[0]
                if len(place_name_geocoding[right_neighbour_name]) == 1:
                    neighbours_state.append([place_name_geocoding[right_neighbour_name][0].state, dist])
            except:
                pass
            try:
                left_neighbour = places_and_poi[first_mention - gap]
                dist = abs(left_neighbour[1].start_char - places_and_poi[first_mention][1].start_char)
                if dist > 300:
                    raise ValueError("neighbour too far away")
                left_neighbour_name = left_neighbour[0]
                if len(place_name_geocoding[left_neighbour_name]) == 1:
                    neighbours_state.append([place_name_geocoding[left_neighbour_name][0].state, dist])
            except:
                pass
        if neighbours_state:
            filtered_geolocations = []
            closest_neighbour_with_exact_geolocation = min(neighbours_state, key=lambda x: x[1])
            filtered_state = closest_neighbour_with_exact_geolocation[0]
            for j in place_name_geocoding[i]:
                if j.state == filtered_state:
                    filtered_geolocations.append(j)
                    break
            if filtered_geolocations:
                place_name_geocoding[i] = filtered_geolocations
            else:
                # try assign target place to more distant neighbour region
                other_neighbour = max(neighbours_state, key=lambda x: x[1])
                filtered_state = other_neighbour[0]
                for j in place_name_geocoding[i]:
                    if j.state == filtered_state:
                        filtered_geolocations.append(j)
                        break
                if filtered_geolocations:
                    place_name_geocoding[i] = filtered_geolocations
    return place_name_geocoding
def locate_based_on_proximity_to_located(place_name_geocoding, min_group=2, max_dist = 150):
    for i in place_name_geocoding:
        neig_dist_ith = dict()
        if len(place_name_geocoding[i]) == 1:
            continue
        for j in place_name_geocoding:
            if len(place_name_geocoding[j]) == 1:
                index = 0
                for a in place_name_geocoding[i]:

                    b = place_name_geocoding[j][0]

                    neig_dist = ll_dist((a.lat, a.long), (b.lat, b.long))

                    if neig_dist_ith.get(index):
                        neig_dist_ith[index].append(neig_dist)
                    else:
                        neig_dist_ith[index] = [neig_dist]
                    index += 1
                        
        possible_geolocations = []
        for key, value in neig_dist_ith.items():
            counter = 0
            for v in value:
                if v <= max_dist:
                    counter += 1
            if counter >= min_group:
                possible_geolocations.append([place_name_geocoding[i][key], counter])
        if possible_geolocations:
            geolocation = max(possible_geolocations, key=lambda x: x[1])
            #print(type(geolocation), geolocation)
            place_name_geocoding[i] = [geolocation[0]]
            
    return place_name_geocoding
def locate_based_on_population(place_name_geocoding):
    for i in place_name_geocoding:
        if len(place_name_geocoding[i]) > 1:
            if any(j.population > 0 for j in place_name_geocoding[i]):
                most_popular = max(place_name_geocoding[i], key=lambda x: x.population)
                place_name_geocoding[i] = [most_popular]
    return place_name_geocoding
def remove_geolocations_not_in_states_mentioned(place_name_geocoding, states):
    for i in place_name_geocoding:
        filtered_geolocations = []
        if len(place_name_geocoding[i]) > 1:
            for j in place_name_geocoding[i]:
                if j.state in states:
                    filtered_geolocations.append(j)
        if filtered_geolocations:
            place_name_geocoding[i] = filtered_geolocations
    return place_name_geocoding
def locate_in_group(place_name_geocoding, min_group_num=3):
    states = dict()
    for i in place_name_geocoding:
        if len(place_name_geocoding[i]) == 1:
            if states.get(place_name_geocoding[i][0].state):
                states[place_name_geocoding[i][0].state] +=1
            else:
                states[place_name_geocoding[i][0].state] = 1
    # choose top states
    states = sorted(list(states.items()), key=lambda x: x[1])

    top_states = [j[0] for j in states if j[1] >= 3]
    for i in place_name_geocoding:
        filtered_geolocations = []
        if len(place_name_geocoding[i]) > 1:
            for j in place_name_geocoding[i]:
                if j.state in top_states:
                     filtered_geolocations.append(j)
            if filtered_geolocations:
                place_name_geocoding[i] = filtered_geolocations
    return place_name_geocoding
def remove_empty_geo_point(place_name_geocoding):
    remove_keys = []
    for i in place_name_geocoding:
        if len(place_name_geocoding[i]) == 0:
            remove_keys.append(i)
    for i in remove_keys:
        del place_name_geocoding[i]
    return place_name_geocoding
def remove_mere_state_names(place_name_geocoding):
    for i in place_name_geocoding:
        state_name = False
        for j in place_name_geocoding[i]:
            if j.place_type == "first-order administrative division":
                state_name = True
        if state_name:
            place_name_geocoding[i] = []
    return place_name_geocoding
def disambiguate(places_geocoded, ne_index, places_and_poi):
    x, y = leave_one_country(places_geocoded)
    x = remove_duplicates_and_leave_states(x, y)
    x = located_based_on_neighbours(x, ne_index, places_and_poi)
    x = locate_in_group(x)
    x = located_based_on_neighbours(x, ne_index, places_and_poi)
    x = locate_based_on_population(x)
    x = locate_in_group(x)
    x = locate_based_on_proximity_to_located(x, max_dist=300)
    x = remove_mere_state_names(x)
    x = remove_empty_geo_point(x)
    print("States mentioned ", y)
    return x<jupyter_output><empty_output><jupyter_text>Visualization<jupyter_code>import altair as alt
from vega_datasets import data as vg_data
import pandas as pd
import geopandas as gpd
import plotly.graph_objects as go
def visualize_points(place_name_geocoding):
    name, lat, long = [], [], []
    for i in place_name_geocoding:
        if len(place_name_geocoding[i]) == 1:
            name.append(i)
            lat.append(place_name_geocoding[i][0].lat)
            long.append(place_name_geocoding[i][0].long)
                
    df = pd.DataFrame(list(zip(name, lat, long)), columns =['Name', 'Lat', "Long"])
#     states = alt.topo_feature(vg_data.us_10m.url, feature='states')
#     gdf = gpd.read_file('https://raw.githubusercontent.com/python-visualization/folium/master/tests/us-states.json', driver='GeoJSON')
#     gdf = gdf[gdf.id=='CA']
    
#     background = alt.Chart(states).mark_geoshape(
#         fill='lightgray',
#         stroke='white'
#     ).project('albersUsa').properties(
#         width=900,
#         height=1000
#     )
    
#     base = alt.Chart(gdf).mark_geoshape(
#     stroke='black', 
#     fill='lightgray').properties(
#         width=700,
#         height=1000
#     )

#     points = alt.Chart(df).mark_circle().encode(
#         longitude='Long:Q',
#         latitude='Lat:Q',
#         color=alt.Color('Name:N', legend=alt.Legend(symbolLimit=0)),
#         size=alt.value(50),
#         tooltip='Name'
#     )

#     return background + points
    fig = go.Figure(data=go.Scattergeo(
        lon = df['Long'],
        lat = df['Lat'],
        text = df['Name'],
        mode = 'markers'
        ))

    fig.update_layout(
            title = 'Geocoded mentioned places',
            geo_scope='usa',
            height=800,
            margin=dict(l=0, r=0, t=50, b=0),
        )
    return fig.show()<jupyter_output><empty_output><jupyter_text>Test page<jupyter_code>def find_places(story_id: int):
    page_url = data[str(story_id)]["article_url"]
    text, links = text_from_html(page_url)
    places_and_poi_processed, places_and_poi, ne_index = get_ne(text)
    lookup_results = existence_check(places_and_poi_processed)
    places_geocoded = unify_lookup_results(lookup_results)
    place_name_geocoding = disambiguate(places_geocoded, ne_index, places_and_poi)
    return ne_index, places_and_poi, places_geocoded, place_name_geocoding
p1_ne_index, p1_places_and_poi, p1_places_geocoded, p1_place_name_geocoding = find_places(1)
p1_places_geocoded
p1_place_name_geocoding = disambiguate(p1_places_geocoded, p1_ne_index, p1_places_and_poi)
[i for i in p1_place_name_geocoding if len(p1_place_name_geocoding[i]) > 1]
"L.A." in [i[0] for i in p1_places_and_poi]
p1_places_geocoded
p1_place_name_geocoding["Palisades Park"][0]
list(p1_place_name_geocoding.keys())
ll_dist((34.0230223,-118.5116491), ( 32.80227, -117.25948))
visualize_points(p1_place_name_geocoding)
p2_ne_index, p2_places_and_poi, p2_places_geocoded, p2_place_name_geocoding = find_places(2)
p2_place_name_geocoding = disambiguate(p2_places_geocoded, p2_ne_index, p2_places_and_poi)
[i for i in p2_place_name_geocoding if len(p2_place_name_geocoding[i]) > 1]
p2_place_name_geocoding["<NAME>"]
list(p2_place_name_geocoding.keys())
visualize_points(p2_place_name_geocoding)
p3_ne_index, p3_places_and_poi, p3_places_geocoded, p3_place_name_geocoding = find_places(3)
[i for i in p3_place_name_geocoding if len(p3_place_name_geocoding[i]) > 1]
p3_places_geocoded
"" in [i[0] for i in p3_places_and_poi]
p3_place_name_geocoding["Frozen Niagara"][0]
ll_dist((35.8277934,-83.5843169), (35.65454, -83.44127))
list(p3_place_name_geocoding.keys())
visualize_points(p3_place_name_geocoding)
p4_ne_index, p4_places_and_poi, p4_places_geocoded, p4_place_name_geocoding = find_places(4)
[i for i in p4_place_name_geocoding if len(p4_place_name_geocoding[i]) > 1]
list(p4_place_name_geocoding.keys())
"Congress Trail" in [i[0] for i in p4_places_and_poi]
p4_places_geocoded
p4_place_name_geocoding["Crescent Meadow Road"]
ll_dist((37.4115211,-119.213161), (37.26217, -119.70098))
visualize_points(p4_place_name_geocoding)
p5_ne_index, p5_places_and_poi, p5_places_geocoded, p5_place_name_geocoding = find_places(5)
[i for i in p5_place_name_geocoding if len(p5_place_name_geocoding[i]) > 1]
visualize_points(p5_place_name_geocoding)
p6_ne_index, p6_places_and_poi, p6_places_geocoded, p6_place_name_geocoding = find_places(6)
[i for i in p6_place_name_geocoding if len(p6_place_name_geocoding[i]) > 1]
visualize_points(p6_place_name_geocoding)
p7_ne_index, p7_places_and_poi, p7_places_geocoded, p7_place_name_geocoding = find_places(7)
[i for i in p7_place_name_geocoding if len(p7_place_name_geocoding[i]) > 1]
visualize_points(p7_place_name_geocoding)
p8_ne_index, p8_places_and_poi, p8_places_geocoded, p8_place_name_geocoding = find_places(8)
[i for i in p8_place_name_geocoding if len(p8_place_name_geocoding[i]) > 1]
visualize_points(p8_place_name_geocoding)
p9_ne_index, p9_places_and_poi, p9_places_geocoded, p9_place_name_geocoding = find_places(9)
[i for i in p9_place_name_geocoding if len(p9_place_name_geocoding[i]) > 1]
visualize_points(p9_place_name_geocoding)
p10_ne_index, p10_places_and_poi, p10_places_geocoded, p10_place_name_geocoding = find_places(10)
[i for i in p10_place_name_geocoding if len(p10_place_name_geocoding[i]) > 1]
visualize_points(p10_place_name_geocoding)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09777611279501414, 'fraction_numerical': 0.01570003065081906, 'mean_word_length': 2.6677491881089184, 'pattern_counts': {'":': 8, '<': 40, '<?xml version=': 0, '>': 62, 'https://': 6, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '644955', 'license': 'MIT', 'n_tokens_mistral': 9972, 'n_tokens_neox': 9426, 'n_words': 1850, 'path': 'src/pipeline.ipynb', 'repo_name': 'kwh44/geoner', 'size': '29374'}