<jupyter_start><jupyter_text>** Let's illustrate how to train different binary classifiers using NLTK text corpus 'movie_reviews'. We will use python packages 'nltk' and 'sklearn'.**<jupyter_code># source: https://www.nltk.org/book/ch06.html
# download movie review dataset
import nltk
nltk.download('movie_reviews')
# import downloaded corpus
# prepare corpus (already tokenized)
from nltk.corpus import movie_reviews
import random<jupyter_output><empty_output><jupyter_text>Data Processing in Boolen format*   List item*   List item<jupyter_code>documents = [(list(movie_reviews.words(fileid)), category) \
             for category in movie_reviews.categories() \
             for fileid in movie_reviews.fileids(category)]
# print out the size of documents
print(len(documents))

# shuffle documents, reorganize the order of documents
random.shuffle(documents)

# split documents into training and test data
doc_train, doc_test = documents[100:], documents[:100]
# print out the size of training and test documents
print(len(doc_train), len(doc_test))
# display category set
categories = set([c for d, c in doc_train])
print(categories)

# display example movie review and its category
print(doc_train[0][0])
print(doc_train[0][1])
# determine vocabulary
# get the word frequency distribution of training data
all_words = nltk.FreqDist(w.lower() for d, c in doc_train for w in d)
# get the top-2000 most common words
word_features = [word for word, num in all_words.most_common(2000)]
# display first 20 words in word_features
print(word_features[:20])

# display the size of all_words
all_word_num = len(all_words)
print(all_word_num)
# display top-k most common words, k in [0, 50) is generated randomly
k = random.randint(0, 50)
print(k)
all_words.pprint(k)
# plot the word frequency distribution of top-k most common words
all_words.plot(k, title='word distribution')

# define a function, return a dict, the type of key is str and value is Boolean
def document_features(document):
  document_words = set(document)
  features = {}
  for word in word_features:
    features[word] = (word in document_words)
  return features
# convert documents into Boolean vectors
train_set = [(document_features(d), c) for (d,c) in doc_train]
test_set = [(document_features(d), c) for (d,c) in doc_test]

# print example features
print(train_set[0][0])<jupyter_output>{',': True, 'the': True, '.': True, 'a': True, 'and': True, 'of': True, 'to': True, "'": True, 'is': True, 'in': True, 's': True, '"': True, 'it': True, 'that': True, '-': True, ')': True, '(': True, 'as': True, 'with': True, 'for': True, 'his': True, 'this': True, 'film': True, 'he': True, 'i': True, 'but': True, 'on': True, 'are': True, 't': True, 'by': True, 'be': True, 'one': True, 'movie': True, 'an': True, 'who': True, 'not': True, 'you': False, 'from': True, 'at': True, 'was': True, 'have': True, 'they': True, 'has': True, 'her': True, 'all': True, 'there': True, '?': True, 'like': True, 'so': True, 'out': True, 'about': True, 'up': True, 'more': False, 'what': True, 'when': True, 'or': True, 'which': True, 'their': True, 'she': True, ':': True, 'some': False, 'just': True, 'can': True, 'if': True, 'we': True, 'him': True, 'into': True, 'even': True, 'only': True, 'no': True, 'than': True, 'good': True, 'time': True, 'most': True, 'its': True, 'will': True, 'story': True, 'would[...]<jupyter_text>Train a Linear Support Vector machine and Random Forest classifier using NLTK and SKlearn<jupyter_code># train an SVM model with training set
import nltk.classify
from sklearn.svm import LinearSVC

# wrapper sklearn classifier using nltk
classifier = nltk.classify.SklearnClassifier(LinearSVC())

# train the LinearSVC classifier using training data
classifier.train(train_set)

# apply the trained SVM model for test set
accuracy = nltk.classify.accuracy(classifier, test_set)
print(accuracy)
# train an RF model with training set
from sklearn.ensemble import RandomForestClassifier

classifier = nltk.classify.SklearnClassifier(RandomForestClassifier())
classifier.train(train_set)

# apply the trained RF model for test set
accuracy = nltk.classify.accuracy(classifier, test_set)
print(accuracy)<jupyter_output>0.84<jupyter_text>Data processing in Float format<jupyter_code># convert documents into tfidf vectors
from sklearn.feature_extraction.text import TfidfVectorizer

tfidf = TfidfVectorizer(stop_words='english')

train_text = [' '.join(d) for (d,c) in doc_train]
train_tfidf = tfidf.fit_transform(train_text)
train_cats = [c for d, c in doc_train]

test_text = [' '.join(d) for (d,c) in doc_test]
test_tfidf = tfidf.transform(test_text)
test_cats = [c for d, c in doc_test]<jupyter_output><empty_output><jupyter_text>Train a Random Forest Classifier using SKlearn<jupyter_code># train an RF model with training set
classifier = RandomForestClassifier()
classifier.fit(train_tfidf, train_cats)
# apply the trained RF model for test set
accuracy = classifier.score(test_tfidf, test_cats)
print(accuracy)<jupyter_output>0.82
{'fraction_non_alphanumeric': 0.13547486033519554, 'fraction_numerical': 0.007581803671189146, 'mean_word_length': 5.9240331491712706, 'pattern_counts': {'":': 1, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '372218', 'license': 'Apache-2.0', 'n_tokens_mistral': 1719, 'n_tokens_neox': 1656, 'n_words': 551, 'path': 'NLP/Classification/NLP_hands_on_1_classification_TFIDF_NLTK (1).ipynb', 'repo_name': 'secretppcdc/secretppcdc.github.com', 'size': '5012'}