<commit_before>from os.path import exists, join
from openelex.base.load import BaseLoader
from openelex.us.md import fetch
import json
import csv

class LoadResults(BaseLoader):
    
    def run(self, year):
        files = self.filenames[year]
        check_cache(files)
        process_files(files)
        
    def check_cache(self, year, files):
        uncached_files = []
        for f in files:
            if not exists(join(self.cache_dir, f['generated_name'])):
                uncached_files.append(f)
        if len(uncached_files) > 0:
            fetch_files(year, uncached_files)
        
    def fetch_files(self, year, files):
        f = fetch.FetchResults()
        f.run(year, files)
        # logging?
    
    def process_files(self, files):
        for f in files:
            with open(join(self.cache_dir, f['generated_name'])) as csvfile:
                reader = csv.DictReader(csvfile)
                for row in reader:
                    # lookup party from party mappings?
                    # lookup office from office mappings?
                    # if office is president, then skip state in lookup, otherwise use
                    office, created = Office.objects.get_or_create()
                    # parse candidate
                    name = parse_name(row['Candidate Name'])
                    # if office is president, then skip state in lookup, otherwise use
                    candidate, created = Candidate.objects.get_or_create(state=self.state.upper(), given_name=name.first, family_name=name.last, name=name.full_name)
                    # build result<commit_msg>nearly finished on md loader<commit_after>from os.path import exists, join
from openelex.base.load import BaseLoader
from openelex.us.md import fetch
from mongoengine import *
from openelex.models import Candidate, Result, Contest
import json
import csv

class LoadResults(BaseLoader):
    
    def run(self, year):
        files = self.filenames[str(year)]
        self.check_cache(str(year), files)
        self.process_files(files)
        
    def check_cache(self, year, files):
        uncached_files = []
        for f in files:
            if not exists(join(self.cache_dir, f['generated_name'])):
                uncached_files.append(f)
        if len(uncached_files) > 0:
            fetch_files(year, uncached_files)
        
    def fetch_files(self, year, files):
        f = fetch.FetchResults()
        f.run(year, files)
        # logging?
    
    def process_files(self, files):
        connect('openelex_results')
        for f in files:
            with open(join(self.cache_dir, f['generated_name'])) as csvfile:
                reader = csv.DictReader(csvfile)
                if 'state_legislative' in f['generated_name']:
                    reporting_level = 'state_legislative'
                    # fix ' LEGS 26'
                    districts = [j for j in reader.fieldnames if j not in ['Vote Type (FOR=1,AGAINST=2)', 'County', 'Candidate Name', 'Party', 'Office Name', 'Office District', 'Winner', 'Write-In?']]
                elif 'county' in f['generated_name']:
                    reporting_level = 'county'
                elif 'precinct' in f['generated_name']:
                    reporting_level = 'precinct'
                else:
                    reporting_level = 'state'
                    jurisdiction = "ocd-division/country:us/state:md"
                for row in reader:
                    # parse candidate - we get full names
                    name = self.parse_name(row['Candidate Name'])
                    # if office is president, then skip state in lookup, otherwise use
                    if row['Office Name'] == 'President - Vice Pres':
                        candidate = Candidate(state='US', given_name=name.first, additional_name=name.middle, family_name=name.last, suffix=name.suffix, name=name.full_name)
                    else:
                        candidate = Candidate(state=self.state.upper(), given_name=name.first, additional_name=name.middle, family_name=name.last, suffix=name.suffix, name=name.full_name)
                    if row['Write-In?'] == 'Y':
                        write_in = True
                    else:
                        write_in = False
                    if reporting_level == 'state_legislative':
                        for district in districts:
                            ocd = "ocd-division/country:us/state:md/sldl:%s" % district.strip().lower()
                            result = Result(ocd_id=ocd, jurisdiction=district.strip(), raw_office=row['Office Name'], reporting_level=reporting_level, candidate=candidate, party=row['Party'], write_in=write_in, total_votes=row[district]).save()
                    elif reporting_level == 'county':
                        jurisdiction = f['name']
                        total_votes = row['Total Votes']
                    elif reporting_level == 'precinct':
                        jurisdiction = str(row['Election District'])+"-"+str(row['Election Precinct'])
                        total_votes = row['Election Night Votes']
                    if not reporting_level == 'state_legislative':
                        result = Result(ocd_id=f['ocd_id'], jurisdiction=jurisdiction, raw_office=row['Office Name'], reporting_level=reporting_level, candidate=candidate, party=row['Party'], write_in=write_in, total_votes=total_votes).save()
        
{'fraction_non_alphanumeric': 0.09983422361392522, 'fraction_numerical': 0.0011051759071652237, 'mean_word_length': 1.9933847850055126, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '733482', 'n_tokens_mistral': 1501, 'n_tokens_neox': 1425, 'n_words': 357}