<jupyter_start><jupyter_text>IntroIn this assignement you will see a little example about*   How to download a dataset*   How to load it into dataframes*   How to use extract basic features from the corpus*   How to train a classifier on those featuresYou are advised to look at the code and understand it.You will be then asked to perform feature extraction and classification yourself.The task that will be addressed is document classification, specifically tasks related to sentiment classification.  Document Classification Tutorial  Preliminary StepsThese are some preliminary steps before addressing the task.Import some basic libraries and set a variable that will be used in multiple steps.<jupyter_code># The libraries we will use are imported here, in case of runtime problems
import os, shutil  #  file management
import sys
import pandas as pd  #  dataframe management
import numpy as np  #  data manipulation
dataset_name = "aclImdb"<jupyter_output><empty_output><jupyter_text>CHANGED: If it is necessary, clean all the data that are in the DATASETS directory and wipe out all the subdirectories.IMPORTANT: DO NOT RUN THIS SECTION UNLESS YOU NEED IT.<jupyter_code>folder = os.getcwd()

print("Current work directory: " + str(folder))

dataset_folder = os.path.join(os.getcwd(), "Datasets")

if not os.path.exists(dataset_folder):
    os.makedirs(dataset_folder)

for filename in os.listdir(dataset_folder):
    file_path = os.path.join(dataset_folder, filename)
    try:
        if os.path.isfile(file_path) or os.path.islink(file_path):
            os.unlink(file_path)
        elif os.path.isdir(file_path):
            shutil.rmtree(file_path)
    except Exception as e:
        print('Failed to delete %s. Reason: %s' % (file_path, e))

print("Cleaned")<jupyter_output>Current work directory: /content
Cleaned<jupyter_text>Dataset DownloadDowload the Imdb movie reviews dataset<jupyter_code>import urllib.request  #  download files
import tarfile  #  unzip files

dataset_folder = os.path.join(os.getcwd(), "Datasets", "Original")

if not os.path.exists(dataset_folder):
    os.makedirs(dataset_folder)

url = 'http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz'

dataset_path = os.path.join(dataset_folder, "Movies.tar.gz")

if not os.path.exists(dataset_path):
    urllib.request.urlretrieve(url, dataset_path)
    print("Successful download")

tar = tarfile.open(dataset_path)
tar.extractall(dataset_folder)
tar.close()
print("Successful extraction")<jupyter_output>Successful download
Successful extraction<jupyter_text>Look at the files you have downloaded to understand the structure of the dataset.  Create the dataframeNow the dataset is loaded into a dataframe to be more accessible.During the creation some data will be printed as an example.<jupyter_code>dataset_name = "aclImdb"

debug = True

dataframe_rows = []

for split in ['train', 'test']:
    for sentiment in ['pos', 'neg']:
        folder = os.path.join(os.getcwd(), "Datasets", "Original", dataset_name, split, sentiment)
        for filename in os.listdir(folder):
            file_path = os.path.join(folder, filename)
            try:
                if os.path.isfile(file_path):
                    # open the file
                    with open(file_path, mode='r', encoding='utf-8') as text_file:
                        # read it and extract informations
                        text = text_file.read()
                        score = filename.split("_")[1].split(".")[0]
                        file_id = filename.split("_")[0]

                        num_sentiment = -1

                        if sentiment == "pos" : num_sentiment = 1
                        elif sentiment == "neg" : num_sentiment = 0

                        # create single dataframe row
                        dataframe_row = {
                            "file_id": file_id,
                            "score": score,
                            "sentiment": num_sentiment,
                            "split": split,
                            "text": text
                        }

                        # print detailed info for the first file
                        if debug:
                            print(file_path)
                            print(filename)
                            print(file_id)
                            print(text)
                            print(score)
                            print(sentiment)
                            print(split)
                            print(dataframe_row)
                            debug = False
                        dataframe_rows.append(dataframe_row)

            except Exception as e:
                print('Failed to process %s. Reason: %s' % (file_path, e))
                sys.exit(0)

folder = os.path.join(os.getcwd(), "Datasets", "Dataframes", dataset_name)
if not os.path.exists(folder):
    os.makedirs(folder)

# transform the list of rows in a proper dataframe
dataframe = pd.DataFrame(dataframe_rows)
dataframe = dataframe[["file_id",
                       "score",
                       "sentiment",
                       "split",
                       "text"]]
dataframe_path = os.path.join(folder, dataset_name + ".pkl")
dataframe.to_pickle(dataframe_path)<jupyter_output>/content/Datasets/Original/aclImdb/train/pos/8460_7.txt
8460_7.txt
8460
Gundam Wing is a fun show. I appreciate it for getting me into Gundam and anime in general. However, after watching its predecessors, such as Mobile Suit Gundam, Zeta Gundam, and even G Gundam, I find Wing to be Gundam Lite.<br /><br />Characters: An aspect long held by Gundam is to have their characters thrust into difficulties and grow into maturity. This does not happen in Wing. Heero is top dog at the beginning, and he's top dog at the end. Personalities do not change, growth is never achieved. The best character is Zechs, who is for all intents and purposes a hero throughout most of the series. But suddenly the series betrays him and turns him into a villain for no apparent reason.<br /><br />Mecha: Wing has great suit designs. The Gundams are super cool, with the Epyon being my favorite. I even consider a few of the OZ suit designs to be on par with some of the classic Zeon suits. But sweet suit designs doesn[...]<jupyter_text>Little analysis of the dataframe<jupyter_code>dataframe_path = os.path.join(os.getcwd(), "Datasets", "Dataframes", dataset_name, dataset_name + ".pkl")
df = pd.read_pickle(dataframe_path)

print("Dataframe structure:")
print(df)
print()


print("Total rows %d" % (len(df)))
print()

column = 'score'
print("Distribution of scores: ")
print(df[column].value_counts())
print()

column = 'sentiment'
print("Distribution of sentiment: ")
print(df[column].value_counts())
print()


for split in ["train", "test"]:
    df1 = df.loc[df['split'] == split]
    column = 'score'
    print("Distribution of scores in %s: " % (split))
    print(df1[column].value_counts())
    print()<jupyter_output>Dataframe structure:
      file_id score  ...  split                                               text
0        8460     7  ...  train  Gundam Wing is a fun show. I appreciate it for...
1        6065    10  ...  train  This movie is horrible- in a 'so bad it's good...
2        7905    10  ...  train  I feel very sorry for people who go to movies ...
3        2836     7  ...  train  Recap: Zandalee is a young woman that feels mo...
4       10468     9  ...  train  Monstervision was a show I grew up with. From ...
...       ...   ...  ...    ...                                                ...
49995    2839     1  ...   test  This movie was the slowest and most boring so ...
49996    3250     1  ...   test  This movie beats everything out there. Well, d...
49997   12007     2  ...   test  I am having a hard time finding the words to e...
49998    9169     1  ...   test  <NAME> has finally run a fairly weak seri...
49999    8988     1  ...   test  I watched this movie for the hot [...]<jupyter_text>A little game: let's create a word cloud for the two sentiments and see if there is something interesting.<jupyter_code>from wordcloud import WordCloud, STOPWORDS
import matplotlib.pyplot as plt

dataframe_path = os.path.join(os.getcwd(), "Datasets", "Dataframes", dataset_name, dataset_name + ".pkl")
df = pd.read_pickle(dataframe_path)

positive_corpus = df.loc[df['sentiment'] == 1]
positive_corpus = positive_corpus["text"].tolist()

negative_corpus = df.loc[df['sentiment'] == 0]
negative_corpus = negative_corpus["text"].tolist()

print("Negative:")
wordcloud1 = WordCloud(width = 3000, height = 2000, collocations=False, stopwords = STOPWORDS).generate(" ".join(negative_corpus))
plt.figure(figsize=(10, 10))
plt.imshow(wordcloud1) 
plt.axis("off");
plt.show()

print("Positive:")
wordcloud2 = WordCloud(width = 3000, height = 2000, collocations=False, stopwords = STOPWORDS).generate(" ".join(positive_corpus))
plt.figure(figsize=(10, 10))
plt.imshow(wordcloud2) 
plt.axis("off");
plt.show()<jupyter_output><empty_output><jupyter_text>Is there something weird about these words? You can repeat the process multiple times to test your observations.Is there something that will influence our approach? Think about it...  Features extractionLet's tackle the Sentiment Analysis task.The scikit-learn library offers *TfidfVectorizer*, a class that performs both tokenization and the creation of the BoW representation (as tf-idf) of a corpus.The class has plenty of options: it can be used also to count n-grams, excluding stop-words, and cutting off most and/or less frequent terms.<jupyter_code>from sklearn.feature_extraction.text import  TfidfVectorizer

dataframe_path = os.path.join(os.getcwd(), "Datasets", "Dataframes", dataset_name, dataset_name + ".pkl")
df = pd.read_pickle(dataframe_path)

# select only the training sentences
df_train = df.loc[df['split'] == "train"]

train_corpus = df_train['text'].tolist()

print("Processing corpus\n")
vectorizer =  TfidfVectorizer()
# tokenization and creation of Bag of Words representation
X_train = vectorizer.fit_transform(train_corpus)

print("Shape of the matrix: (data points, features)")
print(X_train.shape)
print()

# targets for the training set
Y_train = np.array(df_train['sentiment'].tolist())<jupyter_output><empty_output><jupyter_text>Fitting on a corpus, the vectorizer creates an internal vocabulary that will be used to create the BoW representation.The vocabulary is a dictionary that associates to each word a corresponding column in the feature matrix.Do not try to print the whole vocabulary: it is quite large ;)<jupyter_code>print(len(vectorizer.vocabulary_))<jupyter_output><empty_output><jupyter_text>Since the vocabulary will be quite big, by default X will be a *sparse* matrix (see scipy.sparse for more info).<jupyter_code>print(X_train)<jupyter_output><empty_output><jupyter_text>All the scipy tools are compatible with sparse matrices, but if you somehow need the traditional representation it is possible to convert it with the following command. Keep in mind that if the vocabulary and/or the dataset are large, the resulting array will be quite large, possibly occupying all the RAM.```X.toarray()``` It is then possible to parse the test split likewise. Remember to use the transform function and not the fit_transform when working on the test set.<jupyter_code># select only the test sentences
df_test = df.loc[df['split'] == "test"]
test_corpus = df_test['text'].tolist()
X_test = vectorizer.transform(test_corpus)
Y_test = np.array(df_test['sentiment'].tolist())<jupyter_output><empty_output><jupyter_text>Training and TestingIt is now possible to choose one of the many models available in the sci-kit learn library, train it and test it.<jupyter_code># training and testing
from sklearn.svm import LinearSVC

classifier = LinearSVC()

classifier.fit(X_train, Y_train)

Y_pred_train = classifier.predict(X_train)
Y_pred_test = classifier.predict(X_test)
# print results
from sklearn.metrics import classification_report

report_train = classification_report(Y_train, Y_pred_train, target_names=["neg", "pos"])
report_test = classification_report(Y_test, Y_pred_test, target_names=["neg", "pos"])
print("Train")
print(report_train)
print("Test")
print(report_test)
# more advanced tools
# plot precision/recall curve and confusion matrix
from sklearn.metrics import precision_recall_curve, plot_precision_recall_curve, plot_confusion_matrix
import matplotlib.pyplot as plt

disp = plot_precision_recall_curve(classifier, X_test, Y_test)
plot_confusion_matrix(classifier, X_test, Y_test, normalize='true', cmap=plt.cm.Blues, values_format=".2f")<jupyter_output><empty_output><jupyter_text>Now...do it yourself!Now, rather then sentiment classification, try to address the task of score prediction on the same dataset: based on the text of the reviews, try to predict the score assigned by the reviewer.  First partAs first part, experiment with the options of the vectorizer to improve the classification using logistic regression. You are required to look at the documentation of the functions.You can also use or add other pre-processing steps to the text if you feel like it, you are not obliged to use TfidfVectorizer.Obviously, you will need to use the variable names declared in the first block, so to make our evaluation blocks work.Some notes:* Between the problem as regression and the problem as multi-class classification, the priority is the problem as regression. Indeed, the classification perspective does penalize each error in the same way, while the regression perspective considers the difference between the predicted value and the real value (prediction 8 for real value 2 is way worse than prediction 3 for value 2). The classification perspective is a tool to understand the problem better and experiment.* In the multi-class classification, the most important measures are F1s (especially macro and micro), since it is a complex measure that considers more aspects than the simple accuracy. * It is not unusual to not reach a "satisfying" score, especially in the first step: our code makes use of a pretty simple classifier. Our purpose is to show you both the impact of the pre-processing and the choice of the classifier.* There are no threshold values that you have to reach with the measurements to obtain a certain score. The score will be assigned based on the correctness of the methodology and the extent to which different solutions have been explored and discussed.* In each code section, you have to deliver a single approach, but if you have observations or interesting alternative approaches, you can mention them in the .txt file<jupyter_code># write your code here
X_train = None
Y_train = None
X_test = None
Y_test = None<jupyter_output><empty_output><jupyter_text>Now run the code below to train your classifier:<jupyter_code>####################################################################
###################### DO NOT MODIFY THIS PART #####################
from sklearn.linear_model import LogisticRegression

classifier = LogisticRegression(max_iter=1000)

classifier.fit(X_train, Y_train)

Y_pred_test = classifier.predict(X_test)
Y_pred_test_class = np.around(Y_pred_test)

####################################################################<jupyter_output><empty_output><jupyter_text>Now test your results running the code below:<jupyter_code>####################################################################
###################### DO NOT MODIFY THIS PART #####################
from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error
from sklearn.metrics import classification_report, confusion_matrix, plot_confusion_matrix
import matplotlib.pyplot as plt


# task as regression

print("R2 score %f" % (r2_score(Y_test, Y_pred_test)))
print("MAE %f" % (mean_absolute_error(Y_test, Y_pred_test)))
print("MSE %f" % (mean_squared_error(Y_test, Y_pred_test)))
print()

# task as multiclass classification

report = classification_report(Y_test.astype('int'), Y_pred_test_class.astype('int'), labels=[1,2,3,4,7,8,9,10])
print(report)

confusion_matrix = confusion_matrix(Y_test.astype('int'), Y_pred_test_class.astype('int'), labels=[1,2,3,4,7,8,9,10])
print(confusion_matrix)
print()

# fancy confusion matrix
plot_confusion_matrix(classifier, X_test, Y_test, normalize='true', cmap=plt.cm.Blues, values_format=".2f")

####################################################################<jupyter_output><empty_output><jupyter_text>Second partNow experiment both with the processing (the vectorizer) and the classifier to obtain the best result.<jupyter_code># write your code here
X_train = None
Y_train = None
X_test = None
Y_test = None
classifier = None<jupyter_output><empty_output><jupyter_text>Run the code below to train your classifier, then test it with the block that follows.<jupyter_code>####################################################################
###################### DO NOT MODIFY THIS PART #####################
classifier.fit(X_train, Y_train)

Y_pred_train = classifier.predict(X_train)
Y_pred_test = classifier.predict(X_test)
Y_pred_test_class = np.around(Y_pred_test)
####################################################################
####################################################################
###################### DO NOT MODIFY THIS PART #####################
from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error
from sklearn.metrics import classification_report, confusion_matrix, plot_confusion_matrix
import matplotlib.pyplot as plt


# task as regression

print("R2 score %f" % (r2_score(Y_test, Y_pred_test)))
print("MAE %f" % (mean_absolute_error(Y_test, Y_pred_test)))
print("MSE %f" % (mean_squared_error(Y_test, Y_pred_test)))
print()

# task as multiclass classification

report = classification_report(Y_test.astype('int'), Y_pred_test_class.astype('int'), labels=[1,2,3,4,7,8,9,10])
print(report)

confusion_matrix = confusion_matrix(Y_test.astype('int'), Y_pred_test_class.astype('int'), labels=[1,2,3,4,7,8,9,10])
print(confusion_matrix)
print()

# fancy confusion matrix
plot_confusion_matrix(classifier, X_test, Y_test, normalize='true', cmap=plt.cm.Blues, values_format=".2f")
####################################################################<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12429717779354768, 'fraction_numerical': 0.010535509580217262, 'mean_word_length': 4.013683634373289, 'pattern_counts': {'":': 5, '<': 66, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '27620', 'license': 'CC0-1.0', 'n_tokens_mistral': 5354, 'n_tokens_neox': 5107, 'n_words': 1786, 'path': '.ipynb_checkpoints/Assignment1-checkpoint.ipynb', 'repo_name': 'qazim-mucodema/CryptoStonks', 'size': '18324'}