<commit_before>import cPickle as pickle
import math
import parser
import re
import string

from routez.geocoder.models import Road, Intersection
import routez.geocoder.parser as geoparser

def __latlng_dist(src_lat, src_lng, dest_lat, dest_lng):

    if round(src_lat, 4) == round(dest_lat, 4) and \
            round(src_lng, 4) == round(dest_lng, 4):
        return 0.0

    theta = src_lng - dest_lng
    src_lat_radians = math.radians(src_lat)
    dest_lat_radians = math.radians(dest_lat)
    dist = math.sin(src_lat_radians) * math.sin(dest_lat_radians) + \
        math.cos(src_lat_radians) * math.cos(dest_lat_radians) * \
        math.cos(math.radians(theta))
    dist = math.degrees(math.acos(dist)) * (60.0 * 1.1515 * 1.609344 * 1000.0)
    return dist

def __get_interpolated_latlng(coords, length, pct):
    distance_to_travel = pct * length
    distance_travelled = 0
    prevcoord = None
    for coord in coords:
        if prevcoord:
            seg_travel = __latlng_dist(prevcoord[0], prevcoord[1],
                                     coord[0], coord[1])
            if distance_travelled + seg_travel >= distance_to_travel:
                seg_pct = (distance_to_travel - distance_travelled) / \
                    seg_travel
                    
                new_lat = prevcoord[0] + ((coord[0] - prevcoord[0]) * seg_pct)
                new_lng = prevcoord[1] + ((coord[1] - prevcoord[1]) * seg_pct)
                return (new_lat, new_lng)
            distance_travelled += seg_travel

        prevcoord = coord

def get_location(location_str):
    streets = geoparser.parse_address(location_str)
    if len(streets) == 1:
        addr = geoparser.streetAddress.parseString(location_str)

        r = Road.objects.filter(name__iexact=addr.street.name)        
        if addr.street.type:
            r = r.filter(suffix=addr.street.type)
        if addr.street.number:
            r = r.filter(firstHouseNumber__lte=addr.street.number, 
                         lastHouseNumber__gte=addr.street.number)
        if len(r) > 0:
            coords = pickle.loads(str(r[0].coords))
            number = 0.0
            if addr.street.number:
                number = float(addr.street.number)
            percent = float(number - r[0].firstHouseNumber) / \
                float(r[0].lastHouseNumber - r[0].firstHouseNumber)
            return __get_interpolated_latlng(coords, r[0].length, percent)
        else:
            return None

    elif len(streets) == 2:<commit_msg>Remove redundant parsing in geocoder<commit_after>import cPickle as pickle
import math
import parser
import re
import string

from routez.geocoder.models import Road, Intersection
import routez.geocoder.parser as geoparser

def __latlng_dist(src_lat, src_lng, dest_lat, dest_lng):

    if round(src_lat, 4) == round(dest_lat, 4) and \
            round(src_lng, 4) == round(dest_lng, 4):
        return 0.0

    theta = src_lng - dest_lng
    src_lat_radians = math.radians(src_lat)
    dest_lat_radians = math.radians(dest_lat)
    dist = math.sin(src_lat_radians) * math.sin(dest_lat_radians) + \
        math.cos(src_lat_radians) * math.cos(dest_lat_radians) * \
        math.cos(math.radians(theta))
    dist = math.degrees(math.acos(dist)) * (60.0 * 1.1515 * 1.609344 * 1000.0)
    return dist

def __get_interpolated_latlng(coords, length, pct):
    distance_to_travel = pct * length
    distance_travelled = 0
    prevcoord = None
    for coord in coords:
        if prevcoord:
            seg_travel = __latlng_dist(prevcoord[0], prevcoord[1],
                                     coord[0], coord[1])
            if distance_travelled + seg_travel >= distance_to_travel:
                seg_pct = (distance_to_travel - distance_travelled) / \
                    seg_travel
                    
                new_lat = prevcoord[0] + ((coord[0] - prevcoord[0]) * seg_pct)
                new_lng = prevcoord[1] + ((coord[1] - prevcoord[1]) * seg_pct)
                return (new_lat, new_lng)
            distance_travelled += seg_travel

        prevcoord = coord

def get_location(location_str):
    streets = geoparser.parse_address(location_str)
    if len(streets) == 1:
        r = Road.objects.filter(name__iexact=streets[0].name)
        if streets[0].suffix:
            r = r.filter(suffix=streets[0].suffix)
        if streets[0].number:
            r = r.filter(firstHouseNumber__lte=streets[0].number, 
                         lastHouseNumber__gte=streets[0].number)
        if len(r) > 0:
            coords = pickle.loads(str(r[0].coords))
            number = 0.0
            if streets[0].number:
                number = float(streets[0].number)
            percent = float(number - r[0].firstHouseNumber) / \
                float(r[0].lastHouseNumber - r[0].firstHouseNumber)
            return __get_interpolated_latlng(coords, r[0].length, percent)
        else:
            return None

    elif len(streets) == 2:
{'fraction_non_alphanumeric': 0.10442334881842052, 'fraction_numerical': 0.020601898606342154, 'mean_word_length': 2.2514773473407748, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5407505', 'n_tokens_mistral': 1745, 'n_tokens_neox': 1624, 'n_words': 305}