<commit_before>from os.path import dirname, exists, join
from os import listdir
import inspect
import json

from nameparser import HumanName
import csv
import unicodecsv

from .state import StateBase

class BaseLoader(StateBase):
    """
    Base class for loading results data into MongoDB
    Intended to be subclassed in state-specific load.py modules.
    Reads from cached resources inside each state directory.
    """
    
    
    def __init__(self):
        super(BaseLoader, self).__init__()
        #TODO: use datasource.mappings instead
        #self.filenames = json.loads(open(join(self.mappings_dir,'filenames.json'), 'r').read())
        #TODO: use mappings instead
        self.cached_files = listdir(self.cache_dir)

    def run(self):
        raise NotImplementedError()

    #TODO: Migrate name parsing bits to a transforms/name module or function
    def parse_name(self, name):
        return HumanName(name)
        
    def combine_name_parts(self, bits):
        # expects a list of name bits in order
        return " ".join([x.strip() for x in bits])
    
    def jurisdiction_mappings(self, headers):
        "Given a tuple of headers, returns a JSON object of jurisdictional mappings based on OCD ids"
        filename = join(self.mappings_dir, self.state+'.csv')<commit_msg>base loader tweaks. #23<commit_after>from os.path import dirname, exists, join
from os import listdir
import inspect
import json

from nameparser import HumanName
import csv
import unicodecsv

from .state import StateBase

class BaseLoader(StateBase):
    """
    Base class for loading results data into MongoDB
    Intended to be subclassed in state-specific load.py modules.
    Reads from cached resources inside each state directory.

    """

    def __init__(self):
        super(BaseLoader, self).__init__()
        #TODO: use datasource.mappings instead
        #self.filenames = json.loads(open(join(self.mappings_dir,'filenames.json'), 'r').read())

    def run(self):
        raise NotImplementedError()

    #TODO: Migrate name parsing bits to a transforms/name module or function
    def parse_name(self, name):
        return HumanName(name)

    def combine_name_parts(self, bits):
        # expects a list of name bits in order
        return " ".join([x.strip() for x in bits])

    def jurisdiction_mappings(self, headers):
        "Given a tuple of headers, returns a JSON object of jurisdictional mappings based on OCD ids"
        filename = join(self.mappings_dir, self.state+'.csv')
{'fraction_non_alphanumeric': 0.08247011952191236, 'fraction_numerical': 0.0007968127490039841, 'mean_word_length': 3.241554054054054, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2719511', 'n_tokens_mistral': 727, 'n_tokens_neox': 690, 'n_words': 252}