<jupyter_start><jupyter_text>Data Cleaning<jupyter_code>class Preprocesser():
    
    def __init__(self):
        self.table = str.maketrans('', '', string.punctuation)
    
    def remove_hyperlinks(self,text_list):
        corrected_text = []
        for word in text_list:
            if ':' in word and '/' in word:
                continue
            if 'http' not in word:
                corrected_text.append(word)
        return corrected_text
    
    def remove_punctuations(self,text_lst):
        stripped=[]
        for word in text_lst:
            if '.' in word or '?' in word or '!' in word or ',' in word:
                stripped+=[word]
            else:
                stripped +=[ word.translate(self.table)]
        return stripped
    
    def remove_htmlStructure(self,text_lst):     
        stripped=[]
        for word in text_lst:
            if '/>' in word or '<' in word or '·' in word or word.count('.')>=2:
                continue
            else:
                stripped +=[ word]
        return stripped
    
    def remove_otherInf(self,text_lst):
        stripped=[]
        for word in text_lst:
            if '@' in word:
                continue
            elif '(' in word and ')' in word:
                continue
            else:
                stripped +=[ word]
        return stripped
    
    def remove_number(self,text_lst):
        stripped=[]
        for word in text_lst:
            count=0
            for i in range(10):
                if str(i) in word:
                    count+=1
            if count<2:
                stripped +=[word]
        return stripped
    
    def remove_errorPunctuation(self,text_lst):
        stripped=[]
        for word in text_lst:
            if '.' in word:
                fragement=word.split('.')
                
        return stripped
    
    def preprocess(self,text):
        # preprocess pipline
        text_lst = text.split()
        text_lst = self.remove_hyperlinks(text_lst)
        text_lst = self.remove_otherInf(text_lst)
        text_lst = self.remove_htmlStructure(text_lst)
        text_lst = self.remove_number(text_lst)
        text_lst = self.remove_punctuations(text_lst)
        
        return " ".join(text_lst) 
preprocesser=Preprocesser()
Data=Data.dropna(subset=['utterance']).reset_index(drop=True)
newData=Data.copy()
for i in range(len(Data)):
    newData.at[i,'utterance']=preprocesser.preprocess(Data['utterance'][i])
tokenizer = BertTokenizer.from_pretrained('bert-base-cased', do_lower_case=False)
sentence_tokenizer=nltk.data.load('tokenizers/punkt/english.pickle')
newData.loc[Data['diaglogID']==15287,'utterance']
lens=510
index=[]
for i in range(len(newData)):
    text=newData['utterance'][i]
    sentences=sentence_tokenizer.tokenize(text)
    for sentence in sentences:
        token=tokenizer.tokenize(sentence)
        if len(token)>lens:
            print('Drop the index of ',i,len(token))
            index+=[i]
            break
newData=newData.drop(index)
newData=newData.replace({'utterance': ''}, {'utterance': '.'})
newData<jupyter_output><empty_output><jupyter_text>Split<jupyter_code>np.random.seed(1)
lens=len(Data['diaglogID'].unique())
uniqueIndex=Data['diaglogID'].unique()
index=np.random.permutation(lens)
uniqueIndex=uniqueIndex[index]
Train=[]
Valid=[]
Test=[]
TrainP=[]
ValidP=[]
TestP=[]
for i in range(int(np.floor(lens*0.81))):
    Train+=[pd.DataFrame(Data.loc[Data['diaglogID']==uniqueIndex[i],:])]
    TrainP+=[pd.DataFrame(newData.loc[newData['diaglogID']==uniqueIndex[i],:])]
Train=pd.concat(Train)
TrainP=pd.concat(TrainP)
for i in range(int(np.floor(lens*0.81)),int(np.floor(lens*0.91))):
    Valid+=[pd.DataFrame(Data.loc[Data['diaglogID']==uniqueIndex[i],:])]
    ValidP+=[pd.DataFrame(newData.loc[newData['diaglogID']==uniqueIndex[i],:])]
Valid=pd.concat(Valid)
ValidP=pd.concat(ValidP)
for i in range(int(np.floor(lens*0.91)),int(lens)):
    Test+=[pd.DataFrame(Data.loc[Data['diaglogID']==uniqueIndex[i],:])]
    TestP+=[pd.DataFrame(newData.loc[newData['diaglogID']==uniqueIndex[i],:])]
Test=pd.concat(Test)
TestP=pd.concat(TestP)<jupyter_output><empty_output><jupyter_text>Save the Data<jupyter_code>Data.to_csv('data/Data.csv',index=False)
Train.to_csv('data/Train.csv',index=False)
Valid.to_csv('data/Valid.csv',index=False)
Test.to_csv('data/Test.csv',index=False)
TrainP.to_csv('data/Train_Preprocessing.csv',index=False)
ValidP.to_csv('data/Valid_Preprocessing.csv',index=False)
TestP.to_csv('data/Test_Preprocessing.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14586532134239966, 'fraction_numerical': 0.005922351392849309, 'mean_word_length': 2.9964943032427693, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '169395', 'license': 'MIT', 'n_tokens_mistral': 1555, 'n_tokens_neox': 1519, 'n_words': 238, 'path': 'BERT/Preprocessing.ipynb', 'repo_name': 'wtsyang/UserIntentPrediction', 'size': '4559'}