<commit_before>"""
This module is responsible for maintaining geocodes on
listings in Database.
"""
from math import radians

from sqlalchemy import or_
import googlemaps

from ..db import Session
from ..db.models import Apartment
from ..config import GOOGLE_MAPS_API_KEY
from . import celery


def get_rows(num=2500):
    """
    Get rows with missing geocodes from database.
    """
    s = Session()
    result = s.query(Apartment).filter(or_(
        Apartment.latitude==None,
        Apartment.longitude==None))
    ret = []
    for i, a in enumerate(result):
        if i+1 > num:  # i starts at 0
            break
        ret.append(a)
    s.close()
    return ret


def geocode(apartments):
    s = Session()
    gmaps = googlemaps.Client(key=GOOGLE_MAPS_API_KEY)
    for i, apt in enumerate(apartments):
        try:
            loc = gmaps.geocode(apt.address)[0]
        except (IndexError, TypeError) as e:
            # Can't geocode, delete listing
            s.delete(apt)
            continue
        lat = radians(loc['geometry']['location']['lat'])
        lng = radians(loc['geometry']['location']['lng'])
        apt.latitude = lat
        apt.longitude = lng
        s.add(apt)
    s.commit()


@celery.task()
def tasked_geocode(num=2000):
    """
    Geocode num apartments from db.
    """
    geocode(get_rows(num))<commit_msg>flush every 100 records<commit_after>"""
This module is responsible for maintaining geocodes on
listings in Database.
"""
from math import radians

from sqlalchemy import or_
import googlemaps

from ..db import Session
from ..db.models import Apartment
from ..config import GOOGLE_MAPS_API_KEY
from . import celery


def get_rows(num=2500):
    """
    Get rows with missing geocodes from database.
    """
    s = Session()
    result = s.query(Apartment).filter(or_(
        Apartment.latitude==None,
        Apartment.longitude==None))
    ret = []
    for i, a in enumerate(result):
        if i+1 > num:  # i starts at 0
            break
        ret.append(a)
    s.close()
    return ret


def geocode(apartments):
    s = Session()
    gmaps = googlemaps.Client(key=GOOGLE_MAPS_API_KEY)
    for i, apt in enumerate(apartments):
        try:
            loc = gmaps.geocode(apt.address)[0]
        except (IndexError, TypeError) as e:
            # Can't geocode, delete listing
            s.delete(apt)
            continue
        lat = radians(loc['geometry']['location']['lat'])
        lng = radians(loc['geometry']['location']['lng'])
        apt.latitude = lat
        apt.longitude = lng
        s.add(apt)
        if i % 100 == 0:
            s.flush()
    s.commit()


@celery.task()
def tasked_geocode(num=2000):
    """
    Geocode num apartments from db.
    """
    geocode(get_rows(num))
{'fraction_non_alphanumeric': 0.11617806731813246, 'fraction_numerical': 0.010495837857401375, 'mean_word_length': 2.455, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4100367', 'n_tokens_mistral': 910, 'n_tokens_neox': 858, 'n_words': 235}