<commit_before>#!/usr/bin/env python3

import json
import os
import time
import switrs
import sqlite3
import sys

from googlegeocoder import GoogleGeocoder

class CollisionGoecoder():
    def __init__(self, city_directory):
        self.city_directory = city_directory
        self.city_name = os.path.basename(city_directory)
        self.geocoder = GoogleGeocoder()
        self.read_location_database()

    def location_database_path(self):
        return os.path.join(self.city_directory, "locations.json")

    def read_location_database(self):
        with open(self.location_database_path(), 'r') as location_file:
           self.location_database = json.loads(location_file.read())

    def write_location_database(self):
        with open(self.location_database_path(), 'w') as location_file:
           return location_file.write(json.dumps(self.location_database, sort_keys=True))

    def get_location_for_collision(self, collision):
        location_key = '{0} and {1}'.format(collision.primary_road, collision.secondary_road)
        if location_key in self.location_database:
            return self.location_database[location_key]

        try:
            time.sleep(1) # Be kind.
            search = self.geocoder.get('{0}, Oakland, CA'.format(location_key))
            self.location_database[location_key] = [search[0].geometry.location.lat, search[0].geometry.location.lng]
        except:
            return [0, 0]
        return self.location_database[location_key]

    def fill_query_results_location(self, query):
        connection = sqlite3.connect(os.path.join(self.city_directory, "all-collisions.db"))
        try:
            for row in connection.execute('SELECT * FROM collisions WHERE {0};'.format(query)):
                collision = switrs.Collision(row)
                (collision.latitude, collision.longitude) = self.get_location_for_collision(collision)
                print(collision)

            connection.commit()

        finally:
            connection.close()
            self.write_location_database()

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Must specify city directory")
        sys.exit(1)

    CollisionGoecoder(sys.argv[1]).fill_query_results_location('motor_vehicle_with = "G" OR motor_vehicle_with = "B"')
<commit_msg>Actually update the database when fetching locations<commit_after>#!/usr/bin/env python3

import json
import os
import time
import switrs
import sqlite3
import sys

from googlegeocoder import GoogleGeocoder

class CollisionGoecoder():
    def __init__(self, city_directory):
        self.city_directory = city_directory
        self.city_name = os.path.basename(city_directory)
        self.geocoder = GoogleGeocoder()
        self.read_location_database()

    def location_database_path(self):
        return os.path.join(self.city_directory, "locations.json")

    def read_location_database(self):
        with open(self.location_database_path(), 'r') as location_file:
           self.location_database = json.loads(location_file.read())

    def write_location_database(self):
        with open(self.location_database_path(), 'w') as location_file:
           return location_file.write(json.dumps(self.location_database, sort_keys=True))

    def get_location_for_collision(self, collision):
        location_key = '{0} and {1}'.format(collision.primary_road, collision.secondary_road)
        if location_key in self.location_database:
            return self.location_database[location_key]

        try:
            time.sleep(1) # Be kind.
            search = self.geocoder.get('{0}, Oakland, CA'.format(location_key))
            self.location_database[location_key] = [search[0].geometry.location.lat, search[0].geometry.location.lng]
        except:
            return [0, 0]
        return self.location_database[location_key]

    def update_collision_locations_in_database(self, connection, collisions):
        for collision in collisions:
            connection.execute('UPDATE collisions SET latitude=?,longitude=? WHERE id=?', (collision.latitude, collision.longitude, collision.id))
        connection.commit()

    def fill_query_results_location(self, query):
        connection = sqlite3.connect(os.path.join(self.city_directory, "all-collisions.db"))
        collisions = []
        try:
            for row in connection.execute('SELECT * FROM collisions WHERE {0};'.format(query)):
                collision = switrs.Collision(row)
                collisions.append(collision)
                (collision.latitude, collision.longitude) = self.get_location_for_collision(collision)
                print(collision)

            self.update_collision_locations_in_database(connection, collisions)

        finally:
            connection.close()
            self.write_location_database()

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Must specify city directory")
        sys.exit(1)

    CollisionGoecoder(sys.argv[1]).fill_query_results_location('motor_vehicle_with = "G" OR motor_vehicle_with = "B"')

{'fraction_non_alphanumeric': 0.10156095633274057, 'fraction_numerical': 0.005927682276229994, 'mean_word_length': 3.180016515276631, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7457341', 'n_tokens_mistral': 1491, 'n_tokens_neox': 1438, 'n_words': 279}