<commit_before>import csv

from pymongo import MongoClient

class BaseEdX(object):

    def __init__(self, args):
        self.url = args.url
        client = MongoClient(self.url)
        self.db = client[args.db_name]
        self._collections = None
        self.output_directory = args.output_directory
        self.row_limit = args.row_limit
        self.csv_data = None
        self.list_of_headers = None
        
    def generate_csv(self, csv_data, list_of_headers, output_file):
    	"""
    	Genersate csv report from generated data and given list of headers
    	"""
    	self.csv_data = csv_data
    	self.list_of_headers = list_of_headers
    	number_of_rows = len(csv_data) + 1
        if number_of_rows <= self.row_limit:
	    self._write_to_csv(self.output_file)
	else:
	    if number_of_rows % self.row_limit:
		number_of_splits = number_of_rows // self.row_limit + 1
	    else:
		number_of_splits = number_of_rows // self.row_limit
	    for index in xrange(number_of_splits):
	        self._write_to_csv(output_file.split('.')[0] + '_' + str(index) + '.csv', index)
	        
    def _write_to_csv(self, output_file, number_of_splits=0):
    	"""
    	Helper method to write rows to csv files
    	"""
        with open(output_file, 'w') as csv_file:
	    writer = csv.writer(csv_file)
	    writer.writerow(self.list_of_headers)
	    for row in self.csv_data[number_of_splits * self.row_limit : (number_of_splits + 1) * self.row_limit]:
	        # This loop looks for unicode objects and encodes them to ASCII to avoif Unicode errors,
		# for e.g. UnicodeEncodeError: 'ascii' codec can't encode character u'\xf1'
		for index,item in enumerate(row[:]):
		    if type(item) is unicode:
		        row[index] = item.encode('ascii', 'ignore')
		writer.writerow(row)
    @property
    def collections(self):
    	return self._collections
    	
    @collections.setter
    def collections(self, *collections):
    	self._collections = {collection : self.db[collection] for collection in *collections}<commit_msg>Update base.py<commit_after>import csv

from pymongo import MongoClient

class BaseEdX(object):

    def __init__(self, args):
        self.url = args.url
        client = MongoClient(self.url)
        self.db = client[args.db_name]
        self._collections = None
        self.output_directory = args.output_directory
        self.row_limit = args.row_limit
        self.csv_data = None
        self.list_of_headers = None
        
    def generate_csv(self, csv_data, list_of_headers, output_file):
    	"""
    	Genersate csv report from generated data and given list of headers
    	"""
    	self.csv_data = csv_data
    	self.list_of_headers = list_of_headers
    	number_of_rows = len(csv_data) + 1
        if number_of_rows <= self.row_limit:
	    self._write_to_csv(self.output_file)
	else:
	    if number_of_rows % self.row_limit:
		number_of_splits = number_of_rows // self.row_limit + 1
	    else:
		number_of_splits = number_of_rows // self.row_limit
	    for index in xrange(number_of_splits):
	        self._write_to_csv(output_file.split('.')[0] + '_' + str(index) + '.csv', index)
	        
    def _write_to_csv(self, output_file, number_of_splits=0):
    	"""
    	Helper method to write rows to csv files
    	"""
        with open(output_file, 'w') as csv_file:
	    writer = csv.writer(csv_file)
	    writer.writerow(self.list_of_headers)
	    for row in self.csv_data[number_of_splits * self.row_limit : (number_of_splits + 1) * self.row_limit]:
	        # This loop looks for unicode objects and encodes them to ASCII to avoif Unicode errors,
		# for e.g. UnicodeEncodeError: 'ascii' codec can't encode character u'\xf1'
		for index,item in enumerate(row[:]):
		    if type(item) is unicode:
		        row[index] = item.encode('ascii', 'ignore')
		writer.writerow(row)
    @property
    def collections(self):
    	return self._collections
    	
    @collections.setter
    def collections(self, *_collections):
    	self._collections = {collection : self.db[collection] for collection in _collections}
{'fraction_non_alphanumeric': 0.09679012345679013, 'fraction_numerical': 0.002962962962962963, 'mean_word_length': 3.006923837784372, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2455675', 'n_tokens_mistral': 1403, 'n_tokens_neox': 1310, 'n_words': 308}