<commit_before>from __future__ import print_function
from collections import defaultdict
from os import listdir
from os.path import isfile, join

import pandas as pd
import re
import pprint

## Modify your path ##
myPath = './Export/'
## ## ## ## ## ## ## #

# To emulate perl's native ability to dynamically create 
# data structures: Perl Autovivification allows a programmer
# to refer to a structured variable, and arbitrary sub-elements 
# of that structured variable, without expressly declaring the
# existence of variable and its complete structure beforehand.

# More info at: https://en.wikipedia.org/wiki/Autovivification
# Tree = lambda:defaultdict(Tree)
# t = Tree()

# http://stackoverflow.com/questions/635483/
# what-is-the-best-way-to-implement-nested-
# dictionaries-in-python/19829714#19829714
class Vividict(dict):
    def __missing__(self, key):
        value = self[key] = type(self)()
        return value

class FileExtractor():
    def __init__(self, myPath):
        self.myPath = myPath
        self.fileHash = defaultdict(list)
    def getDirectoryFiles(self):
        if self.myPath:
            onlyfiles = [ f for f in listdir(self.myPath) \
                    if isfile(join(self.myPath,f)) ]
        else:
            print ("Path Not Found")
        return onlyfiles
    def printDirectory(self):
        if self.myPath:
            for i in listdir(self.myPath):
                print (i)
            if self.fileHash:
                for k,v in sorted(self.fileHash.items()):
                    print ("%s: %s" % (k,v)) 
    def extractFiles(self, fileName):
        # Regular Expression Patterns
        eff_pat = r'efficien'
        phi0_pat = r'phi=0'
        phi90_pat = r'phi=90'
        theta90_pat = r'theta=90'

        pattern = [theta90_pat, phi0_pat, phi90_pat, eff_pat]
        for p in pattern:
            match = re.search(p, fileName)
            if match:
               self.fileHash[p].append(fileName)
    def returnFileHash(self):
        return self.fileHash
    def getListOfContents(self, fileName):
        arrayList = []
        path = myPath + fileName
        print (path)
        with open(path, 'r') as fin:
            for line in fin:
                arrayList.append(' '.join(line.split()).split())
        return arrayList        
    def generateDataFrameFromFile(self, filename):
        path = myPath + filename
        return pd.read_csv(path, skiprows=2, delimiter='\t', header=0)

class MathFunctions():
    def __init__(self):
        self.tests = Vividict()

    def printVariables(self):
        print ("cut", self.cut)
        print ("plot_data:", self.plot_data) 
        print ("plot_data2:", self.plot_data)
        print ("arr_signal:", self.arr_signal)
        pp = pprint.PrettyPrinter(indent=4)
        pp.pprint(self.tests)
    def setVariables(self, value=""):
        self.pname = value
        self.cut = value
        self.arr_signal = []
        self.extract_frequency = "2440000000"
        self.signalCount = 0
        self.count = 0
        self.threshold = -5.0
        self.totalSignal = 120
        self.plot_data = []
        self.plot_data2 = []
        self.waterFall = []
        self.sortedSignals = []
        self.results = 0
    def extractLines(self, df):
        print (self.threshold, self.threshold)
        self.signalCount = len(df[df.ix[:,2] >= self.threshold])
        print (df)
        for line in lines:
            if len(line) > 1:
                if line[0] == self.extract_frequency:
                    #print ("piece:", line)
                    #print("line[0]:", line[0])
                    #print("line[2]:", line[2],)
                    #cmpV = float(line[2])
                    #print(cmpV, ">=", self.threshold, ":", cmpV >= self.threshold  )

                    self.count += 1
                    if float(line[2]) >= self.threshold:
                        self.signalCount += 1
                    self.arr_signal.append(line[2])
                    if self.cut == 'theta=90':
                        self.plot_data.append("%.2f" % ((float(line[2]) + 15.0) * 5.0))
                    elif self.count <= 60: 
                        self.plot_data.append("%.2f" % (((float(line[2]) + 15.0) * 5.0)))
                    else:
                        self.plot_data2.append("%.2f" % (((float(line[2]) + 15.0) * 5.0)))
    def countSignals(self, index):
        count = 0
        while ( float(self.sortedSignals[count]) >= index):
            count += 1
            if count >= len(self.sortedSignals):<commit_msg>Resolved errors where dataframe couldn't be parsed. simplifying functions<commit_after>from __future__ import print_function
from collections import defaultdict
from os import listdir
from os.path import isfile, join

import pandas as pd
import re
import pprint

## Modify your path ##
myPath = './Export/'
## ## ## ## ## ## ## #

# To emulate perl's native ability to dynamically create 
# data structures: Perl Autovivification allows a programmer
# to refer to a structured variable, and arbitrary sub-elements 
# of that structured variable, without expressly declaring the
# existence of variable and its complete structure beforehand.

# More info at: https://en.wikipedia.org/wiki/Autovivification
# Tree = lambda:defaultdict(Tree)
# t = Tree()

# http://stackoverflow.com/questions/635483/
# what-is-the-best-way-to-implement-nested-
# dictionaries-in-python/19829714#19829714
class Vividict(dict):
    def __missing__(self, key):
        value = self[key] = type(self)()
        return value

class FileExtractor():
    def __init__(self, myPath):
        self.myPath = myPath
        self.fileHash = defaultdict(list)
    def getDirectoryFiles(self):
        if self.myPath:
            onlyfiles = [ f for f in listdir(self.myPath) \
                    if isfile(join(self.myPath,f)) ]
        else:
            print ("Path Not Found")
        return onlyfiles
    def printDirectory(self):
        if self.myPath:
            for i in listdir(self.myPath):
                print (i)
            if self.fileHash:
                for k,v in sorted(self.fileHash.items()):
                    print ("%s: %s" % (k,v)) 
    def extractFiles(self, fileName):
        # Regular Expression Patterns
        eff_pat = r'efficien'
        phi0_pat = r'phi=0'
        phi90_pat = r'phi=90'
        theta90_pat = r'theta=90'

        pattern = [theta90_pat, phi0_pat, phi90_pat, eff_pat]
        for p in pattern:
            match = re.search(p, fileName)
            if match:
               self.fileHash[p].append(fileName)
    def returnFileHash(self):
        return self.fileHash
    def getListOfContents(self, fileName):
        arrayList = []
        path = myPath + fileName
        print (path)
        with open(path, 'r') as fin:
            for line in fin:
                arrayList.append(' '.join(line.split()).split())
        return arrayList        
    def generateDataFrameFromFile(self, filename):
        path = myPath + filename
        return pd.read_csv(path, skiprows=2, delimiter='\t', header=0)

class MathFunctions():
    def __init__(self):
        self.tests = Vividict()

    def printVariables(self):
        print ("cut", self.cut)
        print ("plot_data:", self.plot_data) 
        print ("plot_data2:", self.plot_data)
        print ("arr_signal:", self.arr_signal)
        pp = pprint.PrettyPrinter(indent=4)
        pp.pprint(self.tests)
    def setVariables(self, value=""):
        self.pname = value
        self.cut = value
        self.arr_signal = []
        self.extract_frequency = "2440000000"
        self.signalCount = 0
        self.count = 0
        self.threshold = -5.0
        self.totalSignal = 120
        self.plot_data = []
        self.plot_data2 = []
        self.waterFall = []
        self.sortedSignals = []
        self.results = 0
    def extractLines(self, df):
        copy_df = df
        
        print (df)
        # checks if line[0] == self.extract_frequency
        
        accepted_df = df[df.ix[:,0] == int(self.extract_frequency)] 
        self.count = len(accepted_df)
        # checks if line[2] >= self.threshold and finds length
        self.signalCount = len(accepted_df[accepted_df.ix[:,2] >= self.threshold])
        self.arr_signal = accepted_df.ix[:,2] 
        #for line in lines:
        #    if len(line) > 1:
        #        if line[0] == self.extract_frequency:
        #            #print ("piece:", line)
        #            #print("line[0]:", line[0])
        #            #print("line[2]:", line[2],)
        #            #cmpV = float(line[2])
        #            #print(cmpV, ">=", self.threshold, ":", cmpV >= self.threshold  )

        #            self.count += 1
        #            if float(line[2]) >= self.threshold:
        #                self.signalCount += 1
        #            self.arr_signal.append(line[2])
        #            if self.cut == 'theta=90':
        #                self.plot_data.append("%.2f" % ((float(line[2]) + 15.0) * 5.0))
        #            elif self.count <= 60: 
        #                self.plot_data.append("%.2f" % (((float(line[2]) + 15.0) * 5.0)))
        #            else:
        #                self.plot_data2.append("%.2f" % (((float(line[2]) + 15.0) * 5.0)))
    def countSignals(self, index):
        count = 0
        while ( float(self.sortedSignals[count]) >= index):
            count += 1
            if count >= len(self.sortedSignals):
{'fraction_non_alphanumeric': 0.12490753460847512, 'fraction_numerical': 0.021557645566944943, 'mean_word_length': 1.9621283255086073, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 18, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3695637', 'n_tokens_mistral': 2964, 'n_tokens_neox': 2805, 'n_words': 676}