<commit_before>#!/usr/bin/env python
import datetime as dt
import dateutil.parser as dateparser
import json
import re

FIELDS = set(['accNowDelinq', 'accOpenPast24Mths', 'acceptD', 'addrCity',
 'addrState', 'annualInc', 'avgCurBal', 'bcOpenToBuy', 'bcUtil',
 'chargeoffWithin12Mths', 'collections12MthsExMed', 'creditPullD',
 'delinq2Yrs', 'delinqAmnt', 'desc', 'dti', 'earliestCrLine', 'empLength',
 'empTitle', 'expD', 'expDefaultRate', 'ficoRangeHigh', 'ficoRangeLow',
 'fundedAmount', 'grade', 'homeOwnership', 'id', 'ilsExpD', 'initialListStatus',
 'inqLast6Mths', 'installment', 'intRate', 'investorCount', 'isIncV', 'listD',
 'loanAmount', 'memberId', 'moSinOldIlAcct', 'moSinOldRevTlOp',
 'moSinRcntRevTlOp', 'moSinRcntTl', 'mortAcc', 'mthsSinceLastDelinq',
 'mthsSinceLastMajorDerog', 'mthsSinceLastRecord', 'mthsSinceRecentBc',
 'mthsSinceRecentBcDlq', 'mthsSinceRecentInq', 'mthsSinceRecentRevolDelinq',
 'numAcctsEver120Ppd', 'numActvBcTl', 'numActvRevTl', 'numBcSats', 'numBcTl',
 'numIlTl', 'numOpRevTl', 'numRevAccts', 'numRevTlBalGt0', 'numSats',
 'numTl120dpd2m', 'numTl30dpd', 'numTl90gDpd24m', 'numTlOpPast12m', 'openAcc',
 'pctTlNvrDlq', 'percentBcGt75', 'pubRec', 'pubRecBankruptcies', 'purpose',
 'reviewStatus', 'reviewStatusD', 'revolBal', 'revolUtil', 'serviceFeeRate',
 'subGrade', 'taxLiens', 'term', 'totCollAmt', 'totCurBal', 'totHiCredLim',
 'totalAcc', 'totalBalExMort', 'totalBcLimit', 'totalIlHighCreditLimit',
 'totalRevHiLim'])

STRING_FIELDS = set(["initialListStatus, grade, addrState",
  "subGrade", "homeOwnership", "reviewStatus", "empTitle",
  "purpose", "addrCity"])

_DATE_RE = re.compile(
  r'\d\d\d\d-\d\d-\d\dT[\d:.]+-[\d:]+'
)

# Clases
class Loan:
  """ Represents a listed lending club loan """

  def __init__(self, data):
    """
    data: a dictionary of loan attributes to values
    
    This class excpects the types of values in 'data'to be
    the same as the values generated from 'json.load()'
    """


    if set(data.iterkeys()) != FIELDS:
      raise ValueError

    # Fill us with well formated dates
    for k,v in data.iteritems():
      if isinstance(v, basestring):
        if _DATE_RE.match(v):
          self.__dict__[str(k)] = dateparser.parse(v)
        else:
          self.__dict__[str(k)] = str(v)
      else:
        self.__dict__[str(k)] = v

    # special cases
    self.isIncV = (self.isIncV == 'VERIFIED')



def main():
  pass

if __name__ == '__main__':
  main()<commit_msg>Add CSV parsing to Loan<commit_after>#!/usr/bin/env python
import csv
import datetime as dt
import dateutil.parser as dateparser
import json
import re

FIELDS = set(['accNowDelinq', 'acceptD', 'addrCity', 'addrState', 'annualInc',
  'avgCurBal', 'bcOpenToBuy', 'bcUtil', 'chargeoffWithin12Mths',
  'collections12MthsExMed', 'delinqAmnt', 'desc', 'dti', 'earliestCrLine',
  'empLength', 'empTitle', 'expD', 'ficoRangeHigh', 'ficoRangeLow', 'grade',
  'homeOwnership', 'id', 'initialListStatus', 'installment', 'intRate',
  'isIncV', 'listD', 'memberId', 'moSinOldIlAcct', 'moSinOldRevTlOp',
  'moSinRcntRevTlOp', 'moSinRcntTl', 'mortAcc', 'mthsSinceLastDelinq',
  'mthsSinceLastMajorDerog', 'mthsSinceLastRecord', 'mthsSinceRecentBc',
  'mthsSinceRecentBcDlq', 'mthsSinceRecentInq', 'mthsSinceRecentRevolDelinq',
  'numActvBcTl', 'numActvRevTl', 'numBcSats', 'numBcTl', 'numIlTl',
  'numOpRevTl', 'numRevAccts', 'numRevTlBalGt0', 'numSats', 'numTl120dpd2m',
  'numTl30dpd', 'numTl90gDpd24m', 'numTlOpPast12m', 'openAcc', 'pctTlNvrDlq',
  'percentBcGt75', 'pubRec', 'pubRecBankruptcies', 'purpose', 'revolBal',
  'revolUtil', 'subGrade', 'taxLiens', 'term', 'totCollAmt', 'totCurBal',
  'totHiCredLim', 'totalAcc', 'totalBalExMort', 'totalBcLimit',
  'totalIlHighCreditLimit', 'totalRevHiLim'])

_DATE_RE = re.compile(
  r'\d\d\d\d-\d\d-\d\dT[\d:.]+-[\d:]+'
)

re.sub(r'_(\w)', lambda x: x.group(1).capitalize(), 'total_il_high_credit_limit')

# Clases
class Loan:
  """ Represents a listed lending club loan """

  def __init__(self, data):
    """
    data: a dictionary of loan attributes to values
    
    This class excpects the types of values in 'data'to be
    the same as the values generated from 'json.load()'
    """
    def makeCamel(attr):
      """Make strings for iterator camelCase"""
      return re.sub(r'_(\w)', lambda x: x.group(1).capitalize(), str(attr))


    if set(makeCamel(k) for k in data.iterkeys()) >= FIELDS:
      raise ValueError

    # Fill us with well formated dates
    for k,v in data.iteritems():
      if isinstance(v, basestring):
        if _DATE_RE.match(v):
          self.data[makeCamel(k)] = dateparser.parse(v)
        else:
          self.data[makeCamel(k)] = str(v)
      else:
        self.data[makeCamel(k)] = v

    # special cases
    #self.isIncV = (self.isIncV == 'VERIFIED')

def parse_csv(filename):
  with open(filename) as f:
    return [Loan(data) for data in csv.DictReader(f)]

def main():
  return parse_csv('data/HistoricalLoanData.csv')

if __name__ == '__main__':
  main()
{'fraction_non_alphanumeric': 0.16720064076892271, 'fraction_numerical': 0.009411293552262716, 'mean_word_length': 4.97488038277512, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1926403', 'n_tokens_mistral': 2013, 'n_tokens_neox': 1895, 'n_words': 399}