<commit_before>from os.path import exists, join
import datetime
import re
import unicodecsv

from openelex.base.load import BaseLoader
from openelex.models import Candidate, Result, Contest
from openelex.lib.text import slugify

from .datasource import Datasource


class LoadResults(BaseLoader):

    def run(self, mapping):
        self.source = mapping['generated_filename']
        self.election_id = mapping['election'] #e.g.  md-2012-11-06-general
        self.timestamp = datetime.datetime.now()
        self.datasource = Datasource()

        print("Loading %s" % self.source)
        # Unlike Results, Contest and Candidate metadata is not deleted and reloaded each
        # time, since this metadata is required for multiple file types.
        # We build lookups for this data to optimize Results data loading.
        #print("Building contest and candidate lookups from previously loaded records, if any")
        self.contest_lkup = self._build_lkup(Contest, self.source)
        self.cand_lkup = self._build_lkup(Candidate, self.source)

        # Reload results fresh every time
        result_count = Result.objects.filter(source=self.source).count()
        if result_count > 0:
            print("Deleting %s previously loaded Result documents" % result_count)

        # Load results based on file type
        if '2002' in self.election_id:
            self.load_2002_file(mapping)
        # special case for 2000 primary
        elif '2000' in self.election_id and 'primary' in self.election_id:
            self.load_2000_primary_file(mapping)
        else:
            self.load_non2002_file(mapping)

    # Private methods
    def _build_lkup(self, doc_klass, source):
        """Build lkup for given document class with slug as key

        self_build_lkup(Candidate, mapping['generated_filename'])

        """
        lkup = {}
        objs = doc_klass.objects.filter(source=source)
        for obj in objs:
            lkup[obj.slug] = obj
        return lkup<commit_msg>delete previously loaded results for md #23<commit_after>from os.path import exists, join
import datetime
import re
import unicodecsv

from openelex.base.load import BaseLoader
from openelex.models import Candidate, Result, Contest
from openelex.lib.text import slugify

from .datasource import Datasource


class LoadResults(BaseLoader):

    def run(self, mapping):
        self.source = mapping['generated_filename']
        self.election_id = mapping['election'] #e.g.  md-2012-11-06-general
        self.timestamp = datetime.datetime.now()
        self.datasource = Datasource()

        print("Loading %s" % self.source)
        # Unlike Results, Contest and Candidate metadata is not deleted and reloaded each
        # time, since this metadata is required for multiple file types.
        # We build lookups for this data to optimize Results data loading.
        #print("Building contest and candidate lookups from previously loaded records, if any")
        self.contest_lkup = self._build_lkup(Contest, self.source)
        self.cand_lkup = self._build_lkup(Candidate, self.source)

        # Reload results fresh every time
        result_count = Result.objects.filter(source=self.source).count()
        if result_count > 0:
            print("\tDeleting %s previously loaded Result documents" % result_count)
            Result.objects.filter(source=self.source).delete()

        # Load results based on file type
        if '2002' in self.election_id:
            self.load_2002_file(mapping)
        # special case for 2000 primary
        elif '2000' in self.election_id and 'primary' in self.election_id:
            self.load_2000_primary_file(mapping)
        else:
            self.load_non2002_file(mapping)

    # Private methods
    def _build_lkup(self, doc_klass, source):
        """Build lkup for given document class with slug as key

        self_build_lkup(Candidate, mapping['generated_filename'])

        """
        lkup = {}
        objs = doc_klass.objects.filter(source=source)
        for obj in objs:
            lkup[obj.slug] = obj
        return lkup
{'fraction_non_alphanumeric': 0.07985347985347985, 'fraction_numerical': 0.016605616605616606, 'mean_word_length': 2.8173345759552655, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5411522', 'n_tokens_mistral': 1208, 'n_tokens_neox': 1116, 'n_words': 346}