<jupyter_start><jupyter_text>Load Data<jupyter_code>def load_data():
    data = pd.read_csv('Dataset/All-Pra&Pasca.csv',sep=",")#nama file data
    return data
tweet_df = load_data()
tweet_df.head()
df  = pd.DataFrame(tweet_df[['Tanggal','Screen Name', 'Source', 'Hashtags', 'Banyak Retweet', 'Dif', 'Sentiment', 'Tweets']])
df
#remove user
def remove_pattern(input_txt, pattern):
    r = re.findall(pattern, input_txt)
    for i in r:
        input_txt = re.sub(i, '', input_txt)
    return input_txt   
df['remove_user'] = np.vectorize(remove_pattern)(df['Tweets'], "@[\w]*")
df
#import stopword
from nltk.corpus import stopwords 
#stopwords_indonesia = stopwords.words('indonesian')
# Create Stopword
with open("Kamus/Stopword - PP.txt", "r") as f:
    stopwords_indonesia = f.readline().split()
 
#import sastrawi
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
factory = StemmerFactory()
stemmer = factory.create_stemmer()

#tokenize
from nltk.tokenize import TweetTokenizer
 
# Happy Emoticons
emoticons_happy = set([
    ':-)', ':)', ';)', ':o)', ':]', ':3', ':c)', ':>', '=]', '8)', '=)', ':}',
    ':^)', ':-D', ':D', '8-D', '8D', 'x-D', 'xD', 'X-D', 'XD', '=-D', '=D',
    '=-3', '=3', ':-))', ":'-)", ":')", ':*', ':^*', '>:P', ':-P', ':P', 'X-P',
    'x-p', 'xp', 'XP', ':-p', ':p', '=p', ':-b', ':b', '>:)', '>;)', '>:-)',
    '<3'
    ])
 
# Sad Emoticons
emoticons_sad = set([
    ':L', ':-/', '>:/', ':S', '>:[', ':@', ':-(', ':[', ':-||', '=L', ':<',
    ':-[', ':-<', '=\\', '=/', '>:(', ':(', '>.<', ":'-(", ":'(", ':\\', ':-c',
    ':c', ':{', '>:\\', ';('
    ])

additional_char = set([
    '\nTTTTT', 'T\n', '&amp', '\x',
    '\n', '\n1','\n2','\n3','\n4','\n5', '\n6', '\n7', '\n8', '\n9',
    '\x80', '\x81', '\x82', '\x83', '\x84', '\x85', '\x86', '\x87', '\x88', '\x89', '\x8c', '\x8d', '\x8f',
    '\x9', '\x91', '\x92', '\x93', '\x94', '\x95', '\x96', '\x97', '\x98', '\x99', '\x9c', '\x9d', '\x9e', '\x9f',
    '\xa', '\xa1', '\xa2', '\xa3', '\xa4', '\xa5', '\xa6', '\xa7', '\xa8', '\xa9', '\xaa', '\xac', '\xad', '\xae',
    '\xb', '\xb1', '\xb2', '\xb3', '\xb4', '\xb5', '\xb6', '\xb7', '\xb8', '\xb9', '\xba', '\xab', '\xbc',
    '\xc', '\xc1', '\xc2', '\xe2', '\xf', '\xf0' 
    ])
 
# all emoticons (happy + sad)
emoticons = emoticons_happy.union(emoticons_sad)
 
def clean_tweets(tweet):
    
    #Convert to lower case
    tweet  = tweet.lower()
    
    #Convert www.* or https?://* to ''
    tweet = re.sub('((www\.[^\s]+)|(https?://[^\s]+))','',tweet)
    
    # remove stock market tickers like $GE
    tweet = re.sub(r'\$\w*', '', tweet)
    
    #Remove additional white spaces
    tweet = re.sub('[\s]+', ' ', tweet)
    
    #Replace #word with word
    tweet = re.sub(r'#([^\s]+)', r'\1',tweet)
    
    # remove old style retweet text "RT" dan karakter yang tidak dibutuhkan
    tweet = re.sub(r"b'rt", '', tweet)
    tweet = re.sub(r"b'", '', tweet)
    tweet = re.sub(r'b"', '', tweet)
    
    # remove stock market tickers like $GE
    tweet = re.sub(r'\$\w*', '', tweet)
    
    # remove hashtags
    # only removing the hash # sign from the word
    tweet = re.sub(r'#', '', tweet)
    
    # tokenize tweets
    tokenizer = TweetTokenizer(preserve_case=False, strip_handles=True, reduce_len=True)
    tweet_tokens = tokenizer.tokenize(tweet)
 
    tweets_clean = []    
    for word in tweet_tokens:
        if (word not in stopwords_indonesia and # remove stopwords
              word not in emoticons and # remove emoticons
                    word not in additional_char and #remove add_char
                        word not in string.punctuation): # remove punctuation
            #tweets_clean.append(word)
            stem_word = stemmer.stem(word) # stemming word
            tweets_clean.append(stem_word)
 
    return tweets_clean
df['tweet_clean'] = df['remove_user'].apply(lambda x: clean_tweets(x))
df
def remove_punct(text):
    text  = " ".join([char for char in text if char not in string.punctuation])
    return text
df['Tweet'] = df['tweet_clean'].apply(lambda x: remove_punct(x))
df
df.sort_values("Tanggal", inplace = True)
df
df.drop(df.columns[[8,]], axis = 1, inplace = True)
df
df.drop_duplicates(subset ="Tweet", keep = 'first', inplace = True)
df
df.to_csv('Dataset/All-Pra & Pasca ND Clean.csv',encoding='utf8', index=False)
df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2574712643678161, 'fraction_numerical': 0.020229885057471263, 'mean_word_length': 3.1917148362235066, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '79161', 'license': 'MIT', 'n_tokens_mistral': 1808, 'n_tokens_neox': 1742, 'n_words': 374, 'path': '.ipynb_checkpoints/2. Preprocessing-checkpoint.ipynb', 'repo_name': 'Pramadita/Analisis-Sentimen-Bansos-Random-Forest', 'size': '4350'}