<jupyter_start><jupyter_text>Preparing data for inference<jupyter_code>import pandas as pd
import numpy as np
import html
import re, string, unicodedata
import nltk
import matplotlib.pyplot as plt
from bs4 import BeautifulSoup
from nltk import word_tokenize, sent_tokenize
from nltk.corpus import stopwords
from nltk.stem import LancasterStemmer, WordNetLemmatizer
from nltk.tokenize import RegexpTokenizer

from keras.preprocessing import text, sequence
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.text import text_to_word_sequence
from keras.preprocessing.sequence import pad_sequences<jupyter_output><empty_output><jupyter_text>Pt 1. Drop, Clean & Split<jupyter_code>df = pd.read_csv("data/hn_all_w_sentiment_cleaned_inplace.csv")
df.head()
df.isnull().sum()
df = df[["commentid","cleaned_comment"]]
df.shape
df.isnull().sum()
df = df.dropna(axis=0, how='any', thresh=None, subset=None, inplace=False)
df.isnull().sum()
df = df.fillna("NULL VALUE PLACEHOLDER COMMENT")
df.isnull().sum()
!mkdir data4
a //10
a, b = df.shape
inc = a //10

c = inc*1
d = inc*2
e = inc*3
f = inc*4
g = inc*5
h = inc*6
i = inc*7
j = inc*8
k = inc*9
l = inc*10

df_a = df[0:c]
df_b = df[c:d]
df_c = df[d:e]
df_d = df[e:f]
df_e = df[f:g]
df_f = df[g:h]
df_g = df[h:i]
df_h = df[i:j]
df_i = df[j:k]
df_j = df[k:l]
df_k = df[l:]
df_d.to_csv('data4/hn_cleaned_ready_for_bert_a_11.csv',index=False)
for value in [c,d,e,f,g,h,i,j,k,l]:
    print(value)
df_a.to_csv('data4/hn_cleaned_ready_for_bert_a1.csv',index=False)
df_b.to_csv('data4/hn_cleaned_ready_for_bert_a2.csv',index=False)
df_c.to_csv('data4/hn_cleaned_ready_for_bert_a3.csv',index=False)
df_d.to_csv('data4/hn_cleaned_ready_for_bert_a11.csv',index=False)
df_e.to_csv('data4/hn_cleaned_ready_for_bert_a4.csv',index=False)
df_f.to_csv('data4/hn_cleaned_ready_for_bert_a5.csv',index=False)
df_g.to_csv('data4/hn_cleaned_ready_for_bert_a6.csv',index=False)
df_h.to_csv('data4/hn_cleaned_ready_for_bert_a7.csv',index=False)
df_i.to_csv('data4/hn_cleaned_ready_for_bert_a8.csv',index=False)
df_j.to_csv('data4/hn_cleaned_ready_for_bert_a9.csv',index=False)
df_k.to_csv('data4/hn_cleaned_ready_for_bert_a10.csv',index=False)
df_l.to_csv('data4/hn_cleaned_ready_for_bert_a11.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13514699429574376, 'fraction_numerical': 0.021939447125932425, 'mean_word_length': 7.702290076335878, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '126323', 'license': 'Apache-2.0', 'n_tokens_mistral': 1036, 'n_tokens_neox': 1016, 'n_words': 156, 'path': 'Data Query & Processing/HN-2_BERT_Data_Split.ipynb', 'repo_name': 'toxic-hackers/DataScience', 'size': '2279'}