<commit_before>import io
import re
import os
import csv
import sys
import time
import datetime
import collections
from sets import Set
from optparse import OptionParser


class ProcessDefaulters:

	def __init__(self):
		self.defaultersList = {}

	def run(self, filename):
		""" 
		Reads the text and sets up the defaulters list
		"""
		self.readText(filename)

	def readText(self, filename):
		""" 
		Read the CSV file into the list
		"""
		outputList   = []
		charsPerLine = []
		totalChars   = 0
		startRegex   = re.compile('^[A-Z]+')

		inputFile        = open(filename, 'rb')
		for index, line in enumerate(inputFile):
			nonWSChars = self.getNonWhitespace(line)
			charsPerLine.append(nonWSChars)
			totalChars += nonWSChars

			if nonWSChars > 50 and 'Name' not in line and startRegex.match(line):
				print line
			elif nonWSChars <= 50 and 'Name' not in line and startRegex.match(line):
				print nonWSChars, line

			index += 1

		print 'Average non-whitespace',(totalChars/index)

	def getNonWhitespace(self, line):
		wsRegex = re.compile('\s')
		total   = 0
		for char in line:
			if not wsRegex.match(char):
				total += 1

		return total

	def isCharge(self, line):
		charges = ['ALCOHOL SMUGGLING'
			,'CIGARETTE SMUGGLING'
			,'CLAIMING VAT REPAYMENT(S) TO WHICH NOT ENTITLED'
			,'CLAIMING MEDICAL EXPENSE REPAYMENT(S) TO WHICH NOT ENTITLED'
			,'CLAIMING '
			,'DELIVERING INCORRECT INCOME TAX RETURN(S)'
			,'DELIVERING INCORRECT INCOME VAT RETURN(S)'
			,'DELIVERING INCORRECT'
			,'FAILURE TO HOLD CURRENT LIQUOR LICENCE'
			,'FAILURE TO LODGE INCOME TAX RETURN(S)'
			,'FAILURE TO LODGE CORPORATION TAX RETURN(S)'
			,'FAILURE TO LODGE VAT RETURN(S)'
			,'FAILURE TO LODGE P35 RETURN(S)'
			,'FAILURE TO LODGE P60 RETURN(S)'
			,'FAILURE TO REMIT VAT'
			,'FAILURE TO MAINTAIN BOOKS AND RECORDS'
			,'FAILURE TO HOLD A CURRENT LIQUOR LICENCE'
			,'FAILURE TO PRODUCE BOOKS & RECORDS'
			,'FAILURE TO'
			,'FILING INCORRECT INCOME TAX RETURN(S)'
			,'ILLEGAL BETTING'
			,'ILLEGAL SELLING OF CIGARETTES'
			,'KEEPING UNTAXED TOBACCO FOR SALE'
			,'MISUSE OF MARKED MINERAL OIL'
			,'OIL LAUNDERING'
			,'OIL SMUGGLING'
			,'POSSESSION OF AN UNREGISTERED VEHICLE'
			,'POSSESSION OF COUNTERFEIT ALCOHOL FOR SALE'
			,'POSSESSION OF']

		for charge in charges:
			if charge in line:
				return line.rstrip().lstrip()<commit_msg>Updated Defaulters<commit_after>import io
import re
import os
import csv
import sys
import time
import datetime
import collections
from sets import Set
from optparse import OptionParser


class ProcessDefaulters:

	def __init__(self):
		self.defaultersList = {}

	def run(self, filename):
		""" 
		Reads the text and sets up the defaulters list
		"""
		self.readText(filename)

	def readText(self, filename):
		""" 
		Read the CSV file into the list
		"""
		outputList   = []
		charsPerLine = []
		totalChars   = 0
		startRegex   = re.compile('^[A-Z]+')

		inputFile    = open(filename, 'rb')
		for index, line in enumerate(inputFile):
			nonWSChars = self.getNonWhitespace(line)
			charsPerLine.append(nonWSChars)
			totalChars += nonWSChars

			if nonWSChars > 50 and 'Name' not in line and startRegex.match(line):
				print line
			elif nonWSChars <= 50 and 'Name' not in line and startRegex.match(line):
				print nonWSChars, line

			index += 1

		print 'Average non-whitespace',(totalChars/index)

	def getNonWhitespace(self, line):
		wsRegex = re.compile('\s')
		total   = 0
		for char in line:
			if not wsRegex.match(char):
				total += 1

		return total

	def isCharge(self, line):
		charges = ['ALCOHOL SMUGGLING'
			,'CIGARETTE SMUGGLING'
			,'CLAIMING VAT REPAYMENT(S) TO WHICH NOT ENTITLED'
			,'CLAIMING MEDICAL EXPENSE REPAYMENT(S) TO WHICH NOT ENTITLED'
			,'CLAIMING '
			,'DELIVERING INCORRECT INCOME TAX RETURN(S)'
			,'DELIVERING INCORRECT INCOME VAT RETURN(S)'
			,'DELIVERING INCORRECT RCT RETURN(S)'
			,'DELIVERING INCORRECT'
			,'FAILURE TO HOLD CURRENT LIQUOR LICENCE'
			,'FAILURE TO DELIVER STATEMENT OF AFFAIRS'
			,'FAILURE TO LODGE INCOME TAX RETURN(S)'
			,'FAILURE TO LODGE CORPORATION TAX RETURN(S)'
			,'FAILURE TO LODGE VAT RETURN(S)'
			,'FAILURE TO LODGE P35 RETURN(S)'
			,'FAILURE TO LODGE P60 RETURN(S)'
			,'FAILURE TO REMIT VAT'
			,'FAILURE TO MAINTAIN BOOKS AND RECORDS'
			,'FAILURE TO HOLD A CURRENT LIQUOR LICENCE'
			,'FAILURE TO PRODUCE BOOKS & RECORDS'
			,'FAILURE TO'
			,'FILING INCORRECT INCOME TAX RETURN(S)'
			,'ILLEGAL BETTING'
			,'ILLEGAL SELLING OF CIGARETTES'
			,'KEEPING UNTAXED TOBACCO FOR SALE'
			,'MISUSE OF MARKED MINERAL OIL'
			,'OIL LAUNDERING'
			,'OIL SMUGGLING'
			,'POSSESSION OF AN UNREGISTERED VEHICLE'
			,'POSSESSION OF COUNTERFEIT ALCOHOL FOR SALE'
			,'POSSESSION OF']

		for charge in charges:
			if charge in line:
				return line.rstrip().lstrip()
{'fraction_non_alphanumeric': 0.10083682008368201, 'fraction_numerical': 0.00502092050209205, 'mean_word_length': 3.724308300395257, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2232636', 'n_tokens_mistral': 2045, 'n_tokens_neox': 1718, 'n_words': 530}