<commit_before>import csv
import os
import re
import sys
import datetime

from national_voter_file.transformers.base import (DATA_DIR,
                                                   BasePreparer,
                                                   BaseTransformer)
import usaddress

__all__ = ['default_file', 'StatePreparer', 'StateTransformer']

default_file = 'OK_Sample.csv'

class StatePreparer(BasePreparer):
    state_path = 'ok'
    state_name = 'Oklahoma'
    sep = ","

    def __init__(self, input_path, *args):
        super(StatePreparer, self).__init__(input_path, *args)

        if not self.transformer:
            self.transformer = StateTransformer()

    def process(self):
        reader = self.dict_iterator(self.open(self.input_path))
        for row in reader:
            yield row


class StateTransformer(BaseTransformer):
    date_format = "%m/%d/%Y"

    input_fields = None

    oklahoma_party_map = {
        "DEM": "DEM",
        "REP": "REP",
        "IND": "UN",
        "LIB": "LIB",
        "AE": "AE"
    }

    col_type_dict = BaseTransformer.col_type_dict.copy()
    col_type_dict['PRECINCT_SPLIT'] = set([str, type(None)])

    col_map = {
        'PRECINCT_SPLIT': None
    }

    #### Contact methods #######################################################

    def extract_name(self, input_dict):
        return {
            'TITLE': None,
            'FIRST_NAME': input_dict['FirstName'],
            'MIDDLE_NAME': input_dict['MiddleName'],
            'LAST_NAME': input_dict['LastName'],
            'NAME_SUFFIX': input_dict['Suffix'],
        }

    def extract_email(self, input_dict):
        return {'EMAIL': None}

    def extract_phone_number(self, input_dict):
        return {'PHONE': None}

    def extract_do_not_call_status(self, input_dict):
        return {'DO_NOT_CALL_STATUS': None}

    #### Demographics methods ##################################################

    def extract_gender(self, input_dict):
        return {'GENDER': "U"}

    def extract_race(self, input_dict):
        return {'RACE': "U"}

    def extract_birth_state(self, input_dict):
        return {'BIRTH_STATE': None}

    def extract_birthdate(self, input_dict):
        if len(input_dict['DateOfBirth']) > 0:
            date = self.convert_date(input_dict['DateOfBirth'])
        else:
            date = None
        return {'BIRTHDATE': date, 'BIRTHDATE_IS_ESTIMATE': 'N'}

    def extract_language_choice(self, input_dict):
        return {'LANGUAGE_CHOICE': None}

    #### Address methods #######################################################
<commit_msg>none for gender and party (#175)<commit_after>import csv
import os
import re
import sys
import datetime

from national_voter_file.transformers.base import (DATA_DIR,
                                                   BasePreparer,
                                                   BaseTransformer)
import usaddress

__all__ = ['default_file', 'StatePreparer', 'StateTransformer']

default_file = 'OK_Sample.csv'

class StatePreparer(BasePreparer):
    state_path = 'ok'
    state_name = 'Oklahoma'
    sep = ","

    def __init__(self, input_path, *args):
        super(StatePreparer, self).__init__(input_path, *args)

        if not self.transformer:
            self.transformer = StateTransformer()

    def process(self):
        reader = self.dict_iterator(self.open(self.input_path))
        for row in reader:
            yield row


class StateTransformer(BaseTransformer):
    date_format = "%m/%d/%Y"

    input_fields = None

    oklahoma_party_map = {
        "DEM": "DEM",
        "REP": "REP",
        "IND": "UN",
        "LIB": "LIB",
        "AE": "AE"
    }

    col_type_dict = BaseTransformer.col_type_dict.copy()
    col_type_dict['PRECINCT_SPLIT'] = set([str, type(None)])

    col_map = {
        'PRECINCT_SPLIT': None
    }

    #### Contact methods #######################################################

    def extract_name(self, input_dict):
        return {
            'TITLE': None,
            'FIRST_NAME': input_dict['FirstName'],
            'MIDDLE_NAME': input_dict['MiddleName'],
            'LAST_NAME': input_dict['LastName'],
            'NAME_SUFFIX': input_dict['Suffix'],
        }

    def extract_email(self, input_dict):
        return {'EMAIL': None}

    def extract_phone_number(self, input_dict):
        return {'PHONE': None}

    def extract_do_not_call_status(self, input_dict):
        return {'DO_NOT_CALL_STATUS': None}

    #### Demographics methods ##################################################

    def extract_gender(self, input_dict):
        return {'GENDER': None}

    def extract_race(self, input_dict):
        return {'RACE': None}

    def extract_birth_state(self, input_dict):
        return {'BIRTH_STATE': None}

    def extract_birthdate(self, input_dict):
        if len(input_dict['DateOfBirth']) > 0:
            date = self.convert_date(input_dict['DateOfBirth'])
        else:
            date = None
        return {'BIRTHDATE': date, 'BIRTHDATE_IS_ESTIMATE': 'N'}

    def extract_language_choice(self, input_dict):
        return {'LANGUAGE_CHOICE': None}

    #### Address methods #######################################################

{'fraction_non_alphanumeric': 0.17159763313609466, 'fraction_numerical': 0.000954380606986066, 'mean_word_length': 2.6746143057503504, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6200139', 'n_tokens_mistral': 1609, 'n_tokens_neox': 1532, 'n_words': 317}