<commit_before>import pandas as pd
import numpy as np
import os

randn = np.random.randn

myglobal = np.arange(100)
myglobalidx = 1

def processMG(arg1):
    if arg1 == None:
        return float(0)
    elif 'm' in arg1:
        return float(arg1[:-1])*1048576
    elif 'g' in arg1:
        return float(arg1[:-1])*1048576*1024
    else:
        return float(arg1)

class TopDirParser:
    def __init__(self, start, end, targetdir=None):

        if targetdir != None:
            filelist = os.listdir(targetdir) 

            self.panel_dict = dict()
            for ifile in filelist:
                fs = pd.to_datetime(ifile[:ifile.find('.')])
                if fs >= pd.to_datetime(start) and fs <= pd.to_datetime(end):
                    self.panel_dict[fs] = pd.read_csv(targetdir+ifile,
                                                      sep='\s+',
                                                      skiprows=range(6),
                                                      comment='<defunct>',
                                                      #usecols = ['RES'],
                                                      squeeze=False,
                                                      converters={'RES':processMG})

    def GetItemList(self,item='USER'):
        if not self.panel_dict:
            return None
        else:
            x = set()
            for i in self.panel_dict:
                x |= set(self.panel_dict[i][item].unique())
<commit_msg>Added progress bar<commit_after>import pandas as pd
import numpy as np
import os
import wx
import threading
import time

def processMG(arg1):
    if arg1 == None:
        return float(0)
    elif 'm' in arg1:
        return float(arg1[:-1])*1048576
    elif 'g' in arg1:
        return float(arg1[:-1])*1048576*1024
    else:
        return float(arg1)

class TopDirParser:
    def __init__(self, start, end, targetdir=None):
        self.targetdir = targetdir
        self.filelist = os.listdir(self.targetdir)
        self.start_ts = start
        self.end_ts = end
        
    def LoadData(self, dlg, queue, numfiles):
        self.panel_dict = dict()
        files_loaded = 0
        start_time = time.time()
        for ifile in self.filelist:
            fs = pd.to_datetime(ifile[:ifile.find('.')])
            if fs >= pd.to_datetime(self.start_ts) and fs <= pd.to_datetime(self.end_ts):
                self.panel_dict[fs] = pd.read_csv(self.targetdir+ifile,
                                                  sep='\s+',
                                                  skiprows=range(6),
                                                  comment='<defunct>',
                                                  #usecols = ['RES'],
                                                  squeeze=False,
                                                  converters={'RES':processMG})
                
                files_loaded += 1
                process_rate = (time.time() - start_time)/files_loaded
                message = "Loaded :{0:^6}".format(files_loaded)+" of "+`numfiles`+" at {0:.2f}".format(round(1/process_rate,2))+" Files/Sec "
                if dlg:
                    wx.CallAfter(dlg.Update,files_loaded,message)
                    if dlg.WasCancelled() == True:
                        dlg.EndModal(-1)
                        break
        if queue:
            queue.put(files_loaded)
        return True

    def GetItemList(self,item='USER'):
        if not self.panel_dict:
            return None
        else:
            x = set()
            for i in self.panel_dict:
                x |= set(self.panel_dict[i][item].unique())

{'fraction_non_alphanumeric': 0.10589849108367627, 'fraction_numerical': 0.019478737997256516, 'mean_word_length': 1.270236612702366, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2182019', 'n_tokens_mistral': 1014, 'n_tokens_neox': 970, 'n_words': 218}