<jupyter_start><jupyter_text>Import libraries<jupyter_code>import unicodedata
import contractions
import nltk
import pandas as pd
import numpy as np
import emoji
import string
import re
from autocorrect import Speller
from nltk.tokenize.casual import reduce_lengthening<jupyter_output><empty_output><jupyter_text>Read csv file<jupyter_code>df1 = pd.read_csv('test.csv')
df1.fillna(np.nan)
df1
df2 = pd.read_csv('train.csv')
df2.fillna(np.nan)
df2<jupyter_output><empty_output><jupyter_text>Merge two csv files<jupyter_code>df = df1.append(df2 , ignore_index=True, sort=False)
df
df.info()
df.dtypes<jupyter_output><empty_output><jupyter_text>Remove some columns<jupyter_code># df = df.drop(["user_id","username","tweet_timestamp","lang","sarcasm_label","sarcasm_type"] ,axis=1)
# df<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code>#nltk.download('stopwords')
#nltk.download('punkt')
#nltk.download('wordnet')
#nltk.download('wordnet_ic')
#nltk.download('sentiwordnet')
#remove user
def remove_pattern(input_txt, pattern):
    r = re.findall(pattern, input_txt)
    for i in r:
        input_txt = re.sub(i, '', input_txt)
    return input_txt    
df['remove_username'] = np.vectorize(remove_pattern)(df['text'], "@[\w]*")
def remove(tweet):
    
    #remove numbers
    tweet = re.sub('[0-9]+', '', tweet)
    
    # remove stock market tickers like $GE
    tweet = re.sub(r'\$\w*', '', tweet)
 
    # remove old style retweet text "RT"
    tweet = re.sub(r'^RT[\s]+', '', tweet)
    
    # remove hashtags
    # only removing the hash # sign from the word
    tweet = re.sub(r'#', '', tweet)
    
    return tweet

df['remove_http'] = df['remove_username'].apply(lambda x: remove(x))
df.drop_duplicates(subset ="remove_http", keep = 'first', inplace = True)

#import stopword
from nltk.corpus import stopwords 
english_stop_words = stopwords.words('english')
porter = nltk.stem.PorterStemmer()

spell = Speller()

#tokenize
from nltk.tokenize import TweetTokenizer
 
# Happy Emoticons
emoticons_happy = set([
    ':-)', ':)', ';)', ':o)', ':]', ':3', ':c)', ':>', '=]', '8)', '=)', ':}',
    ':^)', ':-D', ':D', '8-D', '8D', 'x-D', 'xD', 'X-D', 'XD', '=-D', '=D',
    '=-3', '=3', ':-))', ":'-)", ":')", ':*', ':^*', '>:P', ':-P', ':P', 'X-P',
    'x-p', 'xp', 'XP', ':-p', ':p', '=p', ':-b', ':b', '>:)', '>;)', '>:-)',
    '<3'
    ])
 
# Sad Emoticons
emoticons_sad = set([
    ':L', ':-/', '>:/', ':S', '>:[', ':@', ':-(', ':[', ':-||', '=L', ':<',
    ':-[', ':-<', '=\\', '=/', '>:(', ':(', '>.<', ":'-(", ":'(", ':\\', ':-c',
    ':c', ':{', '>:\\', ';('
    ])

# all emoticons (happy + sad)
emoticons = emoticons_happy.union(emoticons_sad)
def clean_tweets(tweet):

    # Replace unusual chars
    tweet = re.sub(r'’', '\'', tweet).strip()
    tweet = re.sub(r'[“”]', '\"', tweet).strip()

    # remove stock market tickers like $GE
    tweet = re.sub(r'\$\w*', '', tweet)

     # Fix contractions  (you're -> you are)
    tweet = contractions.fix(tweet)

    # remove old style retweet text "RT"
    tweet = re.sub(r'^RT[\s]+', '', tweet)

    # Replace unusual chars
    tweet = re.sub(r'’', '\'', tweet)
    tweet = re.sub(r'[“”]', '\"', tweet).strip()

    # remove hyperlinks
    tweet = re.sub(r'https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b'
                      r'([-a-zA-Z0-9()@:%_\+.~#?&//=]*)', '', tweet)

    # Remove Emojis
    text = re.sub("["
                  u"\U0001F600-\U0001F64F"  # emoticons
                  u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                  u"\U0001F680-\U0001F6FF"  # transport & map symbols
                  u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                  u"\U0001F1F2-\U0001F1F4"  # Macau flag
                  u"\U0001F1E6-\U0001F1FF"  # flags
                  u"\U0001F600-\U0001F64F"
                  u"\U00002702-\U000027B0"
                  u"\U000024C2-\U0001F251"
                  u"\U0001f926-\U0001f937"
                  u"\U0001F1F2"
                  u"\U0001F1F4"
                  u"\U0001F620"
                  u"\u200d"
                  u"\u2640-\u2642"
                  "]+", '', tweet)
    tweet = ''.join([char for char in text if char not in emoji.UNICODE_EMOJI]).strip()

    # Remove extra @ " ٪ " + / ) ( = *
    punctuations = '!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~'
    tweet = ''.join([char for char in tweet if char not in punctuations])
 
    # tokenize tweets
    tokenizer = TweetTokenizer(preserve_case=False, strip_handles=True, reduce_len=True)
    tweet_tokens = tokenizer.tokenize(tweet)

    # Remove numbers
    tweet_tokens = [x for x in tweet_tokens if not (x.isdigit() or x[0] == '-' and x[1:].isdigit())]

    # Remove stop words
    tweet_tokens = [word for word in tweet_tokens if word not in english_stop_words]

    # Remove non-ASCII characters from list of tokenized words
    new_words = []
    for word in tweet_tokens:
        new_word = unicodedata.normalize('NFKD', word).encode('ascii', 'ignore').decode('utf-8', 'ignore')
        if new_word != '':
            new_words.append(new_word)
    tweet_tokens = new_words

    # Fix typos
    tweet_tokens = [spell(reduce_lengthening(word)) for word in tweet_tokens]

    # Normalization
    tweet_tokens = [porter.stem(word) for word in tweet_tokens]

    return tweet_tokens


df['tweet_clean_tokenize'] = df['remove_http'].apply(lambda x: clean_tweets(x))

df
#df.drop(df.columns[[1,2,3]], axis = 1, inplace = True)
df.drop_duplicates(subset ="tweet_clean", keep = 'first', inplace = True)
df.to_csv('output.csv',encoding='utf8', index=False)
df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2075538927489756, 'fraction_numerical': 0.034740780331373594, 'mean_word_length': 3.0948212983223926, 'pattern_counts': {'":': 4, '<': 26, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '161221', 'license': 'MIT', 'n_tokens_mistral': 2352, 'n_tokens_neox': 2144, 'n_words': 396, 'path': 'src/phase2-jupyter/main-v2.ipynb', 'repo_name': 'SamadiPour/STSE-Homework', 'size': '5613'}