<commit_before><commit_msg>ExcelTools<commit_after>from pybrain.datasets.supervised import SupervisedDataSet
from pybrain.optimization.randomsearch import RandomSearch
from xlrd import open_workbook 

def readFromExcel(inCols,targetCols, numRows, fileName, offset=0, sheet=0, dataSet=None, conversionFun=None):
    """Populates a given dataset or creates a new SupervisedDataSet from an exccel file.
       
       inCols = array of colum numbers containing the input data colums, colums are indexed from 0
       targetCols = array of colum numbers containing the target data colums, colums are indexed from 0
       numRows = the number of rows ofs data
       fileName= the name of the excel file
       offset = the row the vaild data starts at
       sheet = the sheet of the workbook the data is on, indexed from 0 as it is in xlrd
       dataSet = the dataset to be populated, a SupervisedDataSet if created if it is None
       conversionFun = used to preprocess data.
    """
    book = open_workbook(fileName)
    sheet=book.sheet_by_index(sheet)
    
    if dataSet is None:
        dataSet=SupervisedDataSet(len(inCols),len(targetCols))
    for r in range(offset,(offset+numRows)):
        input=[]
        target=[]
        for inC in inCols:
            input.append(sheet.cell_value(r,inC))

        for tC in targetCols:
            target.append(sheet.cell_value(r,tC))
        try:
            if conversionFun:
                input=[conversionFun(i) for i in input]
                target=[conversionFun(t) for t in target]
                print input,target
        
            dataSet.addSample(input, target)
        except Exception:
            print 'rejected row {}'.format(r)
    return dataSet
{'fraction_non_alphanumeric': 0.0661636680208938, 'fraction_numerical': 0.002901915264074289, 'mean_word_length': 2.699570815450644, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5579443', 'n_tokens_mistral': 492, 'n_tokens_neox': 478, 'n_words': 171}