<jupyter_start><jupyter_text>Clean data<jupyter_code>def clean(text): # use regular expression to remove specific characters
    text = re.sub("\'", " ", text)
    text = re.sub("[^a-zA-Z]"," ",text)
    text = ' '.join(text.split())
    text = text.lower()
    return text
df['clean_overview'] = df['overview'].astype(str).apply(lambda x: clean(x))
df.head()
def word_counts(text, terms):
    all_words = ' '.join([t for t in text]) 
    all_words = all_words.split() 
    
    fdist = nltk.FreqDist(all_words) 
    words_df = pd.DataFrame({'word':list(fdist.keys()), 'count':list(fdist.values())}) 
  
  # selecting top 20 most frequent words 
  
    d = words_df.nlargest(columns="count", n = terms) 
  # visualize words and frequencies
    plt.figure(figsize=(20,30)) 
    ax = sns.barplot(data=d, x= "count", y = "word") 
    ax.set(ylabel = 'Word') 
    plt.show()
word_counts(df['clean_overview'], 200)
stpwrds = set(nltk.corpus.stopwords.words('english'))
def remove_stops(text):
    cleaned = [w for w in text.split() if not w in stpwrds]
    return ' '.join(cleaned)
df['clean_overview'] = df['clean_overview'].astype(str).apply(lambda x: remove_stops(x))
word_counts(df['clean_overview'], 200)
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
tfidf_vectorizer = TfidfVectorizer(max_df=0.8, max_features=10000)
df.columns
X = df['clean_overview']
y = df[['Family', 'Animation',
       'History', 'Documentary', 'Adventure', 'Western', 'Crime', 'Drama',
       'Horror', 'Science Fiction', 'Romance', 'War', 'Mystery', 'Fantasy',
       'Action', 'TV Movie', 'Thriller', 'Comedy', 'Music']]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
tfidf_xtrain = tfidf_vectorizer.fit_transform(X_train)
tfidf_xtest = tfidf_vectorizer.transform(X_test)
from sklearn.svm import SVC

from sklearn.multiclass import OneVsRestClassifier

from sklearn.metrics import classification_report
model = OneVsRestClassifier(SVC(gamma = 'auto',kernel = 'linear'))
model.fit(tfidf_xtrain, y_train)
predictions = model.predict(tfidf_xtest)
print(classification_report(predictions,y_test))<jupyter_output>precision    recall  f1-score   support

           0       0.24      0.81      0.37       106
           1       0.19      0.88      0.31        59
           2       0.03      0.57      0.05         7
           3       0.16      0.80      0.26        10
           4       0.23      0.68      0.34       173
           5       0.03      1.00      0.06         2
           6       0.29      0.70      0.41       193
           7       0.63      0.67      0.65      1456
           8       0.37      0.77      0.50       190
           9       0.29      0.85      0.43       124
          10       0.30      0.67      0.41       250
          11       0.24      0.81      0.36        31
          12       0.06      0.74      0.12        23
          13       0.14      0.76      0.24        63
          14       0.43      0.75      0.55       445
          15       0.00      0.00      0.00         0
          16       0.40      0.69      0.51       520
          17       0.54    [...]<jupyter_text>Trying something different for classifiers<jupyter_code>cm = df.loc[0]
ls = df.loc[1]
def make_genre_list(s):
    genre_list = ['Family', 'Animation',
       'History', 'Documentary', 'Adventure', 'Western', 'Crime', 'Drama',
       'Horror', 'Science Fiction', 'Romance', 'War', 'Mystery', 'Fantasy',
       'Action', 'TV Movie', 'Thriller', 'Comedy', 'Music']
    gen = []
    for g in genre_list:
        if s.loc[g] == 1:
            gen.append(g)
    return gen
make_genre_list(cm)
make_genre_list(ls)
df['genre_list'] = df.apply(lambda row : make_genre_list(row), axis = 1)
from sklearn.preprocessing import MultiLabelBinarizer
multilabel_binarizer = MultiLabelBinarizer()
multilabel_binarizer.fit(df['genre_list'])


y = multilabel_binarizer.transform(df['genre_list'])
X = df['clean_overview']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
tfidf_xtrain = tfidf_vectorizer.fit_transform(X_train)
tfidf_xtest = tfidf_vectorizer.transform(X_test)
model = OneVsRestClassifier(SVC(gamma = 'auto', kernel = 'linear'))
model.fit(tfidf_xtrain, y_train)
predictions = model.predict(tfidf_xtest)
print(classification_report(predictions,y_test))
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import LogisticRegression
logovr = OneVsRestClassifier(LogisticRegression())
logovr.fit(tfidf_xtrain, y_train)
predicted = logovr.predict(tfidf_xtest)
print(classification_report(predicted, y_test))
mbi = multilabel_binarizer.inverse_transform(predicted)

y_test_df = pd.DataFrame()
y_test_df['genres'] = y_test.apply(lambda row : make_genre_list(row), axis = 1)
ytgenres = list(y_test_df['genres'])
for i in range(len(mbi)):
    print(f'{mbi[i]}\t\t\t\t{ytgenres[i]}')
df.head()           
genre_df = df[['Family', 'Animation',
       'History', 'Documentary', 'Adventure', 'Western', 'Crime', 'Drama',
       'Horror', 'Science Fiction', 'Romance', 'War', 'Mystery', 'Fantasy',
       'Action', 'TV Movie', 'Thriller', 'Comedy', 'Music']]
genre_counts = genre_df.apply(pd.Series.value_counts)
genre_counts = genre_counts.loc[1]
gct = genre_counts.transpose()
gct<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1342169124509987, 'fraction_numerical': 0.04834795594549188, 'mean_word_length': 2.857451403887689, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '90929', 'license': 'MIT', 'n_tokens_mistral': 2064, 'n_tokens_neox': 1915, 'n_words': 398, 'path': 'Scripts/.ipynb_checkpoints/DataAnalysis_aaron_TFIDF-checkpoint.ipynb', 'repo_name': 'AaronRiegel/GenrePrediction', 'size': '5357'}