<commit_before>#!/usr/bin/env python

import os
import sys

# Horrible boilerplate - there must be a better way :)
sys.path.append(
    os.path.abspath(
        os.path.dirname(__file__) + '../../..'
    )
)



import csv
import pprint
import re
import warnings

from projects.models import Project
from core.models import Place

import django.core.exceptions
from django.contrib.gis.geos import Point

def main():
    pi = ProjectImporter()
    pi.import_from_csv_file( sys.argv[1] )


class ProjectImporter(object):
    constituency_qs = Place.objects.filter(kind__slug='constituency')
    not_found_constituencies = set()
    
    # Use this to rename the constituencies to suit what we have in our database
    constituency_renames = {
        # "Alego Usonga": 'Alego',
        # "Buret", # is a division
        # "Kabete", # not found
        # "Kajiado South", # found 'North' and 'Central'
        # "Kipiripiri",
        # "Maragua", # not found
        # "Mukwurweini", # 'Mukurweini' division
        "Cherengany":              "Cherengani",
        "Dagoreti":                "Dagoretti",
        "Igembe North (Ntonyiri)": "Igembe North",
        "Kisumu Rural":            "Kisumi Rural",
        "Kisumu Town East":        "Kisumi Town East",
        "Kisumu Town West":        "Kisumi Town West",
        "Mt. Elgon":               "Mt Elgon",
        "North Mugirango":         "North Mugirango Borabu",
        "North-Horr":              "North Horr",
        "Ol'Kalou":                "Ol' Kalou",
    }
  
    def import_from_csv_file( self, csv_file_name ):
        """Import data from a csv file with the expected fieldnames"""
        
        csv_file = open(csv_file_name)
        
        self.reader = csv.DictReader( csv_file )        
        
        for row in self.reader:
            # try:
            self.import_row( row )
            # except Exception as inst:
            #     print( "Issue %s on csv line %u: %s" % ( repr(inst), self.reader.line_num, inst ))
            # 
            # # break
        
        if len( self.not_found_constituencies ):
            print "The following constituencies could not be found"
            pprint.pprint( self.not_found_constituencies )
        
        
    def import_row(self, row):

        # print "Adding %s" % row['Project name']

        # Numbers exports CSV using 'Nil' for empty fields. Change to empty string.
        for key, value in row.items():
            if value == 'Nil': row[key] = ''
        
        # change money values to floats
        for key in ['estimated cost','Total Amount']:
            try:
                row[key] = float( row[key] )
            except:
                row[key] = 0
        
        data = dict(
            cdf_index           = int(row['index']),
            project_name        = row['Project name'],
            location_name       = row['Location'],
            sector              = row['Sector'],
            mtfe_sector         = row['MTFE Sector'],
            econ1               = row['Econ1'],
            econ2               = row['Econ2'],
            activity_to_be_done = row['activity to bedone'],
            expected_output     = row['expected output'],
            status              = row['implementation status'],
            remarks             = row['remarks'][:400],
            estimated_cost      = row['estimated cost'],
            total_cost          = row['Total Amount'],
            first_funding_year = None,
        )
        
        # find and add the constituency (do this by name rather than location)
        try:
            constituency_name = row['Constituency']
            if constituency_name in self.constituency_renames:
                constituency_name = self.constituency_renames[constituency_name]
            constituency = self.constituency_qs.get(name=constituency_name)
            data['constituency'] = constituency
        except Place.DoesNotExist:
            self.not_found_constituencies.add( row['Constituency'])
            # warnings.warn("Could not find constituency '%s' on csv line '%u'" % ( row['Constituency'], self.reader.line_num ))
            return None
        
        # Establish the first year that there was funding
        for year in range(2000, 2020):
            key = "%u-%u" % ( year, year+1 )
            amount = re.sub( r'\D', '', row.get(key, '0') )
            if int(amount):
                data['first_funding_year'] = year
                break

        # create the location from coordinates given
        try:
            number_regex = re.compile(r'([-\d\.]+)')<commit_msg>Fix for '' not being int()able<commit_after>#!/usr/bin/env python

import os
import sys

# Horrible boilerplate - there must be a better way :)
sys.path.append(
    os.path.abspath(
        os.path.dirname(__file__) + '../../..'
    )
)



import csv
import pprint
import re
import warnings

from projects.models import Project
from core.models import Place

import django.core.exceptions
from django.contrib.gis.geos import Point

def main():
    pi = ProjectImporter()
    pi.import_from_csv_file( sys.argv[1] )


class ProjectImporter(object):
    constituency_qs = Place.objects.filter(kind__slug='constituency')
    not_found_constituencies = set()
    
    # Use this to rename the constituencies to suit what we have in our database
    constituency_renames = {
        # "Alego Usonga": 'Alego',
        # "Buret", # is a division
        # "Kabete", # not found
        # "Kajiado South", # found 'North' and 'Central'
        # "Kipiripiri",
        # "Maragua", # not found
        # "Mukwurweini", # 'Mukurweini' division
        "Cherengany":              "Cherengani",
        "Dagoreti":                "Dagoretti",
        "Igembe North (Ntonyiri)": "Igembe North",
        "Kisumu Rural":            "Kisumi Rural",
        "Kisumu Town East":        "Kisumi Town East",
        "Kisumu Town West":        "Kisumi Town West",
        "Mt. Elgon":               "Mt Elgon",
        "North Mugirango":         "North Mugirango Borabu",
        "North-Horr":              "North Horr",
        "Ol'Kalou":                "Ol' Kalou",
    }
  
    def import_from_csv_file( self, csv_file_name ):
        """Import data from a csv file with the expected fieldnames"""
        
        csv_file = open(csv_file_name)
        
        self.reader = csv.DictReader( csv_file )        
        
        for row in self.reader:
            # try:
            self.import_row( row )
            # except Exception as inst:
            #     print( "Issue %s on csv line %u: %s" % ( repr(inst), self.reader.line_num, inst ))
            # 
            # # break
        
        if len( self.not_found_constituencies ):
            print "The following constituencies could not be found"
            pprint.pprint( self.not_found_constituencies )
        
        
    def import_row(self, row):

        # print "Adding %s" % row['Project name']

        # Numbers exports CSV using 'Nil' for empty fields. Change to empty string.
        for key, value in row.items():
            if value == 'Nil': row[key] = ''
        
        # change money values to floats
        for key in ['estimated cost','Total Amount']:
            try:
                row[key] = float( row[key] )
            except:
                row[key] = 0
        
        data = dict(
            cdf_index           = int(row['index']),
            project_name        = row['Project name'],
            location_name       = row['Location'],
            sector              = row['Sector'],
            mtfe_sector         = row['MTFE Sector'],
            econ1               = row['Econ1'],
            econ2               = row['Econ2'],
            activity_to_be_done = row['activity to bedone'],
            expected_output     = row['expected output'],
            status              = row['implementation status'],
            remarks             = row['remarks'][:400],
            estimated_cost      = row['estimated cost'],
            total_cost          = row['Total Amount'],
            first_funding_year = None,
        )
        
        # find and add the constituency (do this by name rather than location)
        try:
            constituency_name = row['Constituency']
            if constituency_name in self.constituency_renames:
                constituency_name = self.constituency_renames[constituency_name]
            constituency = self.constituency_qs.get(name=constituency_name)
            data['constituency'] = constituency
        except Place.DoesNotExist:
            self.not_found_constituencies.add( row['Constituency'])
            # warnings.warn("Could not find constituency '%s' on csv line '%u'" % ( row['Constituency'], self.reader.line_num ))
            return None
        
        # Establish the first year that there was funding
        for year in range(2000, 2020):
            key = "%u-%u" % ( year, year+1 )
            amount = re.sub( r'\D', '', row.get(key, '0') )
            if len(amount) and int(amount):
                data['first_funding_year'] = year
                break

        # create the location from coordinates given
        try:
            number_regex = re.compile(r'([-\d\.]+)')
{'fraction_non_alphanumeric': 0.10776374041680165, 'fraction_numerical': 0.004103228593024511, 'mean_word_length': 1.7321533923303836, 'pattern_counts': {'":': 22, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5136826', 'n_tokens_mistral': 2747, 'n_tokens_neox': 2670, 'n_words': 699}