<commit_before>import csv
from rapidsms import log as rlog
log = rlog.Logger(level='Debug')
from optparse import make_option
import os

from django.conf import settings
from django.contrib.gis.gdal import DataSource
from django.contrib.gis.gdal.error import OGRException
from django.contrib.gis.geos import Point

from django.core.management.base import NoArgsCommand, CommandError
from django.db import connection, transaction

from apps.locate.models import *

DATA_DIR = 'data/centerline'

class Command(NoArgsCommand):
    help = 'Import centerline road data into the database using the Django models.'

    option_list = NoArgsCommand.option_list + (
        make_option('-c', '--clear', action='store_true', dest='clear',
            help='Clear all centerline data from the DB.'),
        make_option('-t', '--test', action='store_true', dest='test',
            help='Load test centerline data.'),
        )

    def handle_noargs(self, **options):
        if options['clear']:
            log.write('', 'info', 'Clearing all centerline data.')
            Intersection.objects.all().delete()
            Block.objects.all().delete()
            Road.objects.all().delete()
        
        if options['test']:
            shapefile = os.path.join(DATA_DIR, 'test/test.shp')
        else:
            shapefile = os.path.join(DATA_DIR, 'full/Transportation.shp')
            
        ds = DataSource(shapefile)
        layer = ds[0]
        
        for feature in layer:
            road_name = feature.get('STREET_NAM')
            road_prefix_direction = feature.get('PRE_DIR')
            road_type = feature.get('STREET_TYP')
            road_suffix_direction = feature.get('SUF_DIR')
            from_cross_road = feature.get('F_CROSS')
            to_cross_road = feature.get('T_CROSS')
            
            try:
                linestring = feature.geom
            except OGRException:
                continue
            
            # The vast majority of these unnammed roads are at O'Hare.
            # If they don't have names then people can't refer to them anyway
            # So we just skip them.
            if not road_name:
                continue
            
            # Create road for this feature
            road = self.get_or_create_road(
                road_prefix_direction, road_name, road_type, road_suffix_direction)
            
            # Create a block for this feature
            self.create_block_for_feature(feature, road)
        
            # Create roads from intersecting features
            if from_cross_road:
                f_road = self.get_road_for_intersection(from_cross_road)
                
                if f_road:
                    location = Point(linestring.coords[0], srid=9102671)
                    self.create_intersection(road, f_road, location)
            
            if to_cross_road:
                t_road = self.get_road_for_intersection(to_cross_road)
                
                if t_road:
                    location = Point(linestring.coords[-1], srid=9102671)
                    self.create_intersection(road, t_road, location)
        
    def get_or_create_road(self, road_prefix_direction, road_name, road_type, road_suffix_direction):
        """
        Condense boiler-plate for retrieving and creating roads.
        """
        full_name = Road.make_full_name(
            road_prefix_direction, road_name, road_type, road_suffix_direction)
        try:
            road = Road.objects.get(full_name=full_name)
        except Road.DoesNotExist:
            road = Road.objects.create(
                full_name=full_name,
                prefix_direction=road_prefix_direction,
                name=road_name,
                road_type=road_type,
                suffix_direction=road_suffix_direction
                )
                
        return road
        
    def get_road_for_intersection(self, cross):
        """
        Creates a road from the centerline F_CROSS or T_CROSS strings,
        if those strings indicate an intersection w/ another road.
        """
        bits = cross.split('|')[1:] # Trim address number
        bits = [bit.strip() for bit in bits] # Cleanup whitespace
        
        if bits[1].strip():
            road = self.get_or_create_road(*bits)
        else:
            # Road connects to another block of the same road w/o intersection
            # or the Road just ends
            road = None
            
        return road
        
    def create_block_for_feature(self, feature, road):  
        """
        Creates a block from a given feature.
        """     
        from_addr_left = feature.get('L_F_ADD')
        to_addr_left = feature.get('L_T_ADD')
        from_addr_right = feature.get('R_F_ADD')
        to_addr_right = feature.get('R_T_ADD')
        
        try:
            linestring = feature.geom
        except OGRException:
            return
        
        block_number = self.get_block_number(
            from_addr_left, to_addr_left, from_addr_right, to_addr_right)
            
        # Get center-point of block
        location = self.estimate_point_along_linestring(linestring, 0.50)
        <commit_msg>Bug fix to put block locations in the same projection as the intersections.<commit_after>import csv
from rapidsms import log as rlog
log = rlog.Logger(level='Debug')
from optparse import make_option
import os

from django.conf import settings
from django.contrib.gis.gdal import DataSource
from django.contrib.gis.gdal.error import OGRException
from django.contrib.gis.geos import GEOSGeometry, Point

from django.core.management.base import NoArgsCommand, CommandError
from django.db import connection, transaction

from apps.locate.models import *

DATA_DIR = 'data/centerline'

class Command(NoArgsCommand):
    help = 'Import centerline road data into the database using the Django models.'

    option_list = NoArgsCommand.option_list + (
        make_option('-c', '--clear', action='store_true', dest='clear',
            help='Clear all centerline data from the DB.'),
        make_option('-t', '--test', action='store_true', dest='test',
            help='Load test centerline data.'),
        )

    def handle_noargs(self, **options):
        if options['clear']:
            log.write('', 'info', 'Clearing all centerline data.')
            Intersection.objects.all().delete()
            Block.objects.all().delete()
            Road.objects.all().delete()
        
        if options['test']:
            shapefile = os.path.join(DATA_DIR, 'test/test.shp')
        else:
            shapefile = os.path.join(DATA_DIR, 'full/Transportation.shp')
            
        ds = DataSource(shapefile)
        layer = ds[0]
        
        for feature in layer:
            road_name = feature.get('STREET_NAM')
            road_prefix_direction = feature.get('PRE_DIR')
            road_type = feature.get('STREET_TYP')
            road_suffix_direction = feature.get('SUF_DIR')
            from_cross_road = feature.get('F_CROSS')
            to_cross_road = feature.get('T_CROSS')
            
            try:
                linestring = feature.geom
            except OGRException:
                continue
            
            # The vast majority of these unnammed roads are at O'Hare.
            # If they don't have names then people can't refer to them anyway
            # So we just skip them.
            if not road_name:
                continue
            
            # Create road for this feature
            road = self.get_or_create_road(
                road_prefix_direction, road_name, road_type, road_suffix_direction)
            
            # Create a block for this feature
            self.create_block_for_feature(feature, road)
        
            # Create roads from intersecting features
            if from_cross_road:
                f_road = self.get_road_for_intersection(from_cross_road)
                
                if f_road:
                    location = Point(linestring.coords[0], srid=9102671)
                    self.create_intersection(road, f_road, location)
            
            if to_cross_road:
                t_road = self.get_road_for_intersection(to_cross_road)
                
                if t_road:
                    location = Point(linestring.coords[-1], srid=9102671)
                    self.create_intersection(road, t_road, location)
        
    def get_or_create_road(self, road_prefix_direction, road_name, road_type, road_suffix_direction):
        """
        Condense boiler-plate for retrieving and creating roads.
        """
        full_name = Road.make_full_name(
            road_prefix_direction, road_name, road_type, road_suffix_direction)
        try:
            road = Road.objects.get(full_name=full_name)
        except Road.DoesNotExist:
            road = Road.objects.create(
                full_name=full_name,
                prefix_direction=road_prefix_direction,
                name=road_name,
                road_type=road_type,
                suffix_direction=road_suffix_direction
                )
                
        return road
        
    def get_road_for_intersection(self, cross):
        """
        Creates a road from the centerline F_CROSS or T_CROSS strings,
        if those strings indicate an intersection w/ another road.
        """
        bits = cross.split('|')[1:] # Trim address number
        bits = [bit.strip() for bit in bits] # Cleanup whitespace
        
        if bits[1].strip():
            road = self.get_or_create_road(*bits)
        else:
            # Road connects to another block of the same road w/o intersection
            # or the Road just ends
            road = None
            
        return road
        
    def create_block_for_feature(self, feature, road):  
        """
        Creates a block from a given feature.
        """     
        from_addr_left = feature.get('L_F_ADD')
        to_addr_left = feature.get('L_T_ADD')
        from_addr_right = feature.get('R_F_ADD')
        to_addr_right = feature.get('R_T_ADD')
        
        try:
            linestring = feature.geom
        except OGRException:
            return
        
        block_number = self.get_block_number(
            from_addr_left, to_addr_left, from_addr_right, to_addr_right)
            
        # Get center-point of block
        location = GEOSGeometry(self.estimate_point_along_linestring(linestring, 0.50), srid=9102671)
        
{'fraction_non_alphanumeric': 0.08047945205479452, 'fraction_numerical': 0.0048515981735159815, 'mean_word_length': 1.8850164654226125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1882276', 'n_tokens_mistral': 3000, 'n_tokens_neox': 2865, 'n_words': 696}