<jupyter_start><jupyter_text>Functions<jupyter_code>def count_chars(text):
    return len(text)


def count_words(text):
    return len(text.split())


def count_capital_chars(text):
    count=0
    for i in text:
        if i.isupper():
            count+=1
    return count


def count_capital_words(text):
    return sum(map(str.isupper,text.split()))


def count_punctuations(text):
    punctuations= string.punctuation
    d=dict()
    for i in punctuations:
        d[str(i)+' count']=text.count(i)
    return d 


# def count_words_in_quotes(text):
#     x = re.findall("'.'|"."", text)
#     count=0
#     if x is None:
#         return 0
#     else:
#         for i in x:
#             t=i[1:-1]
#             count+=count_words(t)
#         return count
    

def count_sent(text):
    return len(nltk.sent_tokenize(text))


def count_unique_words(text):
    return len(set(text.split()))


def count_htags(text):
    x = re.findall(r'(#w[A-Za-z0-9]*)', text)
    return len(x) 


def count_mentions(text):
    x = re.findall(r'(@w[A-Za-z0-9]*)', text)
    return len(x)


def count_stopwords(text):
    stop_words = set(stopwords.words('english'))  
    word_tokens = word_tokenize(text)
    stopwords_x = [w for w in word_tokens if w in stop_words]
    return len(stopwords_x)
data = '/Users/fttiunjani/gemastik/practice/dataset/dataset-idsa-master/labelled-sentiment-copy.csv'
df = pd.read_csv(data)
df.head()
df.info()
df['char_count'] = df["Tweet"].apply(lambda x:count_chars(x))
df['word_count'] = df["Tweet"].apply(lambda x:count_words(x))
df['sent_count'] = df["Tweet"].apply(lambda x:count_sent(x))
df['capital_char_count'] = df["Tweet"].apply(lambda x:count_capital_chars(x))
df['capital_word_count'] = df["Tweet"].apply(lambda x:count_capital_words(x))
# df['quoted_word_count'] = df["tweet"].apply(lambda x:count_words_in_quotes(x))
df['stopword_count'] = df["Tweet"].apply(lambda x:count_stopwords(x))
df['unique_word_count'] = df["Tweet"].apply(lambda x:count_unique_words(x))
df['htag_count'] = df["Tweet"].apply(lambda x:count_htags(x))
df['mention_count'] = df["Tweet"].apply(lambda x:count_mentions(x))
df['punct_count'] = df["Tweet"].apply(lambda x:count_punctuations(x))
df['avg_wordlength'] = df['char_count']/df['word_count']
df['avg_sentlength'] = df['word_count']/df['sent_count']
df['unique_vs_words'] = df['unique_word_count']/df['word_count']
df['stopwords_vs_words'] = df['stopword_count']/df['word_count']
# Calculating average word length
df['avg_wordlength'] = df['char_count']/df['word_count']

# Calculating average sentence length
df['avg_sentlength'] = df['word_count']/df['sent_count']

# unique words vs word count feature
df['unique_vs_words'] = df['unique_word_count']/df['word_count']

# Stopwords count vs words counts feature
df['stopwords_vs_words'] = df['stopword_count']/df['word_count']
df.head()
to_replace = dict(zip([-1, 0, 1],['negatif', 'netral', 'positif']))
df.replace({'sentimen':to_replace}, inplace=True)
df.head()
# General Pre-processing 
def remove_links(tweet):
    '''Takes a string and removes web links from it'''
    tweet = re.sub(r'http\S+', '', tweet) # remove http links
    tweet = re.sub(r'bit.ly/\S+', '', tweet) # rempve bitly links
    tweet = tweet.strip('[link]') # remove [links]
    return tweet

def remove_users(tweet):
    '''Takes a string and removes retweet and @user information'''
    tweet = re.sub('(RT\s@[A-Za-z]+[A-Za-z0-9-_]+)', '', tweet) # remove retweet
    tweet = re.sub('(@[A-Za-z]+[A-Za-z0-9-_]+)', '', tweet) # remove tweeted at
    return tweet

my_punctuation = '!"$%&\'()*+,-./:;<=>?[\\]^_`{|}~•@'
def preprocess(sent):
    sent = remove_users(sent)
    sent = remove_links(sent)
    sent = sent.lower() # lower case
    sent = re.sub('['+my_punctuation + ']+', ' ', sent) # strip punctuation
    sent = re.sub('\s+', ' ', sent) #remove double spacing
    sent = re.sub('([0-9]+)', '', sent) # remove numbers
    sent_token_list = [word for word in sent.split(' ')]
    sent = ' '.join(sent_token_list)
    return sent
df.Tweet = df.Tweet.apply(lambda x: preprocess(x))
# Using TF-IDF
vectorizer = TfidfVectorizer()
vectorized = vectorizer.fit_transform(df['Tweet']).toarray()
print(vectorized.shape)
df_tfidf = pd.DataFrame(vectorized)
df_tfidf
features = [feature for feature in df.columns if feature !='Tweet' and feature !='sentimen' and feature != 'punct_count']
features
X = pd.merge(df_tfidf, df[features], left_index=True, right_index=True)
y = df['sentimen']
X.head()
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.8, random_state=123)
type(X_train)
X_train.info()
print(f'tweet length for train = {len(X_train)}, sentimen length for train = {len(y_train)}')
print(f'tweet length for test = {len(X_test)}, sentimen length for train = {len(y_test)}')
rf_classifier = RandomForestClassifier(n_estimators = 1000, min_samples_split = 15, random_state = 42)
rf_classifier.fit(X_train, y_train)
rf_prediction = rf_classifier.predict(X_test)
print("Accuracy => ", round(accuracy_score(rf_prediction, y_test)*100, 2))
print("\nRandom Forest Classifier results: \n")
print(classification_report(y_test, rf_prediction, target_names = ['negatif', 'netral', 'positif']))
import joblib
# save
joblib.dump(rf_classifier, "random_forest_sentimen_classifier.joblib")

# load
# loaded_rf = joblib.load("my_random_forest.joblib")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17437325905292478, 'fraction_numerical': 0.006685236768802228, 'mean_word_length': 5.127417519908987, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '325380', 'license': 'MIT', 'n_tokens_mistral': 2057, 'n_tokens_neox': 1994, 'n_words': 378, 'path': 'indonesian-sentiment-analysis.ipynb', 'repo_name': 'SokKanaTorajd/nlp-practices', 'size': '5385'}