<commit_before>import re, pickle, random, os
import pandas as pd
from datetime import datetime
from data_processing import split_wrd, InteractiveAnswer, space_fill, colorit, _in_list
from dateutil.relativedelta import relativedelta

BOARDER_LENGTH = 40

class Quest():
    def __init__(self,q,sel=None,ta=None,args={}):
        self.q = q
        self.sel = sel
        self.ta = ta
        self.args = args


class QuestForm(pd.Series):
    def __init__(self,*args,**kwargs):
        super(QuestForm,self).__init__(*args,**kwargs)

    def append(self,to_append,*args,**kwargs):
        return QuestForm(super(QuestForm,self).append(pd.Series(to_append),ignore_index=True,*args,**kwargs))


class QuestFormTextLoader():
    def __init__(self,questpattern,qpattern,selpattern=None,tapattern=None,argpattern={}):
        self.questpattern = questpattern
        self.qpattern = qpattern
        self.selpattern = selpattern
        self.tapattern = tapattern
        self.argpattern = dict(argpattern)
        self.is_cached = False

    def get_cached_qf(self,togo='Curdata.data'):
        if togo in os.listdir():
            if InteractiveAnswer('Cached data found.Continue?',yes_or_no=True).get():
                with open(togo,'rb') as f: return pickle.load(f)
        return
                
    def _load(self,queststr):
        questform = QuestForm()
        for quest in re.findall(self.questpattern,queststr):
            qitem = re.findall(self.qpattern,quest)
            selitem = re.findall(self.selpattern,quest) if self.selpattern else None
            taitem = re.findall(self.tapattern,quest) if self.tapattern else None
            argitem = [(patnam,re.findall(self.argpattern(patnam),quest)) \
                    for patnam in self.argpattern] if self.argpattern else None
            questform = questform.append(Quest(q=qitem,sel=selitem,ta=taitem,args=argitem))
        return questform

    def load(self,queststr):
        qf = self.get_cached_qf()
        if type(qf) != type(None): 
            self.is_cached = True
            return qf

        if 'MainData.data' in os.listdir():
            with open('MainData.data','rb') as f: qf = pickle.load(f)
        else:
            qf = self._load(queststr)
            with open('MainData.data','wb') as f: pickle.dump(qf, f)

        return qf


class QuestFormExcelLoader(QuestFormTextLoader):
    def __init__(self,qcol,selcol=None,tacol=None,argcol={}):
        super(QuestFormExcelLoader,self).__init__(None,qcol,selcol,tacol,argcol)

    def _load(self,questdf):
        if type(questdf) == str: questdf = pd.read_excel(questdf)
        questform = QuestForm()
        for q in range(len(questdf)):
            quest = questdf.ix[q]
            qitem = quest[self.qpattern]
            selitem = quest[self.selpattern] if self.selpattern else None
            taitem = quest[self.tapattern] if self.tapattern else None
            argitem = {pat:quest[self.argpattern[pat]] for pat in self.argpattern} if self.argpattern else {}

            qitem = [qitem] if type(qitem)==str else list(qitem)
            selitem = [selitem] if type(selitem)==str else list(selitem)
            taitem = [taitem] if type(taitem)==str  else list(taitem)

            questform = questform.append(Quest(q=qitem,sel=selitem,ta=taitem,args=argitem))
        return questform


class BeginQuestForm():
    def __init__(self,qf,arrange='qast',storage='l|w',no_score=False,input_manner=None,no_filter=False):
        self.qf = qf
        self.starttime = datetime.now()
        self.correct = []
        self.wrong = []
        self.arrange = arrange
        self.storage = storage
        self.no_score = no_score
        self.input_manner = input_manner
        self.status = []
        self.no_filter = no_filter

    def selchap(self,qf):
        return qf

    def oninit(self):
        if InteractiveAnswer('Randomize?',yes_or_no=True).get():
            random.shuffle(self.arranged_index)
        print('\n','='*BOARDER_LENGTH,'\n')
        print(space_fill(self.starttime.strftime('%Y-%m-%d %H:%M:%S'),BOARDER_LENGTH))
        print(space_fill('Find %d questions.'%(self.length),BOARDER_LENGTH))
        print(space_fill('start test.',BOARDER_LENGTH))
        print('\n','='*BOARDER_LENGTH,'\n')

    def _report(self):
        print('\n\n','='*BOARDER_LENGTH,'\n')
        usedtime = relativedelta(datetime.now(),self.starttime)
        print(space_fill('Total Time: %d hours, %d minutes, %d seconds'\
                %(usedtime.hours,usedtime.minutes,usedtime.seconds)\
                ,BOARDER_LENGTH))
        if self.no_score: pass
        elif len(self.correct)+len(self.wrong) != 0:
            c = len(self.correct)
            w = len(self.wrong)
            print('Correct: ',c)
            print('Wrong: ',w)
            print('Score: %.2f'%(c/(c+w)*100))
            print('\n','-'*BOARDER_LENGTH,'\n')
            self.show_status(usedtime.hours)
        print('\n','='*BOARDER_LENGTH,'\n')


    def onkill(self):
        print('\n\n','='*BOARDER_LENGTH,'\n')
        print(space_fill('Interrupted',BOARDER_LENGTH))
        self._report()
        self.qf.index = range(len(self.qf))
        self.store_data(level=self.storage)
        return

    def onfinish(self):
        print('\n\n','='*BOARDER_LENGTH,'\n')
        print(space_fill('Finished',BOARDER_LENGTH))
        self._report()
        self.store_data(level=self.storage)
        return

    def store_data(self,togo='Curdata.data',torevise='Wrongdata.data',level='l|w'):
        l = [i for i in self.qf.index if not (_in_list(i,self.correct) | _in_list(i,self.wrong))]

        togoindex = []
        for i,j in zip('cwl',[self.correct,self.wrong,l]):
            if i in level.split('|')[0]: togoindex += j
        qf = self.qf[togoindex]
        qf.index = range(len(qf))
        if len(qf) != 0:
            with open(togo,'wb') as f:
                pickle.dump(qf,f)
        else:
            try: os.remove(togo)
            except: pass
        
        if len(level.split('|')) > 1:
            toreviseindex = []
            for i,j in zip('cwl',[self.correct,self.wrong,l]):
                if i in level.split('|')[1]: toreviseindex += j
            qf = self.qf[toreviseindex]
            if len(qf) != 0:
                if torevise not in os.listdir():
                    with open(torevise,'wb') as f:
                        qf.index = range(len(qf))
                        pickle.dump(qf,f)
                else:
                    with open(torevise,'rb') as f:
                        wrongdata = pickle.load(f)
                    with open(torevise,'wb') as f:
                        wrongdata = wrongdata.append(qf)
                        wrongdata.index = range(len(wrongdata))
                        pickle.dump(wrongdata,f)
        return

    def raise_quest(self,quest):
        ans = None
        for a in self.arrange:
            if re.findall('^'+a,'quest'):
                self.raise_q(quest)
            elif re.findall('^'+a,'args'):
                if not quest.args: continue
                for k in quest.args:<commit_msg>sort before save data<commit_after>import re, pickle, random, os
import pandas as pd
from datetime import datetime
from data_processing import split_wrd, InteractiveAnswer, space_fill, colorit, _in_list
from dateutil.relativedelta import relativedelta

BOARDER_LENGTH = 40

class Quest():
    def __init__(self,q,sel=None,ta=None,args={}):
        self.q = q
        self.sel = sel
        self.ta = ta
        self.args = args


class QuestForm(pd.Series):
    def __init__(self,*args,**kwargs):
        super(QuestForm,self).__init__(*args,**kwargs)

    def append(self,to_append,*args,**kwargs):
        return QuestForm(super(QuestForm,self).append(pd.Series(to_append),ignore_index=True,*args,**kwargs))


class QuestFormTextLoader():
    def __init__(self,questpattern,qpattern,selpattern=None,tapattern=None,argpattern={}):
        self.questpattern = questpattern
        self.qpattern = qpattern
        self.selpattern = selpattern
        self.tapattern = tapattern
        self.argpattern = dict(argpattern)
        self.is_cached = False

    def get_cached_qf(self,togo='Curdata.data'):
        if togo in os.listdir():
            if InteractiveAnswer('Cached data found.Continue?',yes_or_no=True).get():
                with open(togo,'rb') as f: return pickle.load(f)
        return
                
    def _load(self,queststr):
        questform = QuestForm()
        for quest in re.findall(self.questpattern,queststr):
            qitem = re.findall(self.qpattern,quest)
            selitem = re.findall(self.selpattern,quest) if self.selpattern else None
            taitem = re.findall(self.tapattern,quest) if self.tapattern else None
            argitem = [(patnam,re.findall(self.argpattern(patnam),quest)) \
                    for patnam in self.argpattern] if self.argpattern else None
            questform = questform.append(Quest(q=qitem,sel=selitem,ta=taitem,args=argitem))
        return questform

    def load(self,queststr):
        qf = self.get_cached_qf()
        if type(qf) != type(None): 
            self.is_cached = True
            return qf

        if 'MainData.data' in os.listdir():
            with open('MainData.data','rb') as f: qf = pickle.load(f)
        else:
            qf = self._load(queststr)
            with open('MainData.data','wb') as f: pickle.dump(qf, f)

        return qf


class QuestFormExcelLoader(QuestFormTextLoader):
    def __init__(self,qcol,selcol=None,tacol=None,argcol={}):
        super(QuestFormExcelLoader,self).__init__(None,qcol,selcol,tacol,argcol)

    def _load(self,questdf):
        if type(questdf) == str: questdf = pd.read_excel(questdf)
        questform = QuestForm()
        for q in range(len(questdf)):
            quest = questdf.ix[q]
            qitem = quest[self.qpattern]
            selitem = quest[self.selpattern] if self.selpattern else None
            taitem = quest[self.tapattern] if self.tapattern else None
            argitem = {pat:quest[self.argpattern[pat]] for pat in self.argpattern} if self.argpattern else {}

            qitem = [qitem] if type(qitem)==str else list(qitem)
            selitem = [selitem] if type(selitem)==str else list(selitem)
            taitem = [taitem] if type(taitem)==str  else list(taitem)

            questform = questform.append(Quest(q=qitem,sel=selitem,ta=taitem,args=argitem))
        return questform


class BeginQuestForm():
    def __init__(self,qf,arrange='qast',storage='l|w',no_score=False,input_manner=None,no_filter=False):
        self.qf = qf
        self.starttime = datetime.now()
        self.correct = []
        self.wrong = []
        self.arrange = arrange
        self.storage = storage
        self.no_score = no_score
        self.input_manner = input_manner
        self.status = []
        self.no_filter = no_filter

    def selchap(self,qf):
        return qf

    def oninit(self):
        if InteractiveAnswer('Randomize?',yes_or_no=True).get():
            random.shuffle(self.arranged_index)
        print('\n','='*BOARDER_LENGTH,'\n')
        print(space_fill(self.starttime.strftime('%Y-%m-%d %H:%M:%S'),BOARDER_LENGTH))
        print(space_fill('Find %d questions.'%(self.length),BOARDER_LENGTH))
        print(space_fill('start test.',BOARDER_LENGTH))
        print('\n','='*BOARDER_LENGTH,'\n')

    def _report(self):
        print('\n\n','='*BOARDER_LENGTH,'\n')
        usedtime = relativedelta(datetime.now(),self.starttime)
        print(space_fill('Total Time: %d hours, %d minutes, %d seconds'\
                %(usedtime.hours,usedtime.minutes,usedtime.seconds)\
                ,BOARDER_LENGTH))
        if self.no_score: pass
        elif len(self.correct)+len(self.wrong) != 0:
            c = len(self.correct)
            w = len(self.wrong)
            print('Correct: ',c)
            print('Wrong: ',w)
            print('Score: %.2f'%(c/(c+w)*100))
            print('\n','-'*BOARDER_LENGTH,'\n')
            self.show_status(usedtime.hours)
        print('\n','='*BOARDER_LENGTH,'\n')


    def onkill(self):
        print('\n\n','='*BOARDER_LENGTH,'\n')
        print(space_fill('Interrupted',BOARDER_LENGTH))
        self._report()
        self.qf.index = range(len(self.qf))
        self.store_data(level=self.storage)
        return

    def onfinish(self):
        print('\n\n','='*BOARDER_LENGTH,'\n')
        print(space_fill('Finished',BOARDER_LENGTH))
        self._report()
        self.store_data(level=self.storage)
        return

    def store_data(self,togo='Curdata.data',torevise='Wrongdata.data',level='l|w'):
        l = [i for i in self.qf.index if not (_in_list(i,self.correct) | _in_list(i,self.wrong))]

        togoindex = []
        for i,j in zip('cwl',[self.correct,self.wrong,l]):
            if i in level.split('|')[0]: togoindex += j
        togoindex.sort()
        qf = self.qf[togoindex]
        qf.index = range(len(qf))
        if len(qf) != 0:
            with open(togo,'wb') as f:
                pickle.dump(qf,f)
        else:
            try: os.remove(togo)
            except: pass
        
        if len(level.split('|')) > 1:
            toreviseindex = []
            for i,j in zip('cwl',[self.correct,self.wrong,l]):
                if i in level.split('|')[1]: toreviseindex += j
            toreviseindex.sort()
            qf = self.qf[toreviseindex]
            if len(qf) != 0:
                if torevise not in os.listdir():
                    with open(torevise,'wb') as f:
                        qf.index = range(len(qf))
                        pickle.dump(qf,f)
                else:
                    with open(torevise,'rb') as f:
                        wrongdata = pickle.load(f)
                    with open(torevise,'wb') as f:
                        wrongdata = wrongdata.append(qf)
                        wrongdata.index = range(len(wrongdata))
                        pickle.dump(wrongdata,f)
        return

    def raise_quest(self,quest):
        ans = None
        for a in self.arrange:
            if re.findall('^'+a,'quest'):
                self.raise_q(quest)
            elif re.findall('^'+a,'args'):
                if not quest.args: continue
                for k in quest.args:
{'fraction_non_alphanumeric': 0.13346390598768887, 'fraction_numerical': 0.0016787912702853946, 'mean_word_length': 2.26564641388762, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2661114', 'n_tokens_mistral': 4729, 'n_tokens_neox': 4553, 'n_words': 828}