<jupyter_start><jupyter_text>Deep Learning Try<jupyter_code>from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from sklearn.metrics import accuracy_score, log_loss
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Flatten
from keras.layers.embeddings import Embedding
train_df , test_df = train_test_split(more_data,test_size = 0.2 , random_state = 23)
token = Tokenizer(num_words = None , char_level = False)
token.fit_on_texts(more_data.text)
train_seq = pad_sequences(token.texts_to_sequences(train_df.text),maxlen = 50)
test_seq = pad_sequences(token.texts_to_sequences(test_df.text),maxlen = 50)
train_lab = np.array(train_df.is_offensive) 
test_lab = np.array(test_df.is_offensive) 
emb = Embedding(len(token.word_index)+1,128,input_length = 50)
dense = Dense(1,activation = 'sigmoid')
model = Sequential()
model.add(emb)
model.add(Flatten())
model.add(dense)
model.compile(optimizer = 'sgd',loss = 'binary_crossentropy',metrics = ['accuracy'])
print(model.summary())
model.fit(train_seq,train_lab,epochs = 5)
loss,accuracy = model.evaluate(test_seq,test_lab)
print(round(accuracy * 100,2),model)
names = ['I hope you die']
name_token = pad_sequences(token.texts_to_sequences(names),maxlen = 50)
pred = model.predict_classes(name_token)
pred<jupyter_output><empty_output><jupyter_text>OPEN AI TEST<jupyter_code>import openai
openai.api_key = "<KEY>"
response = openai.Completion.create(engine="davinci", prompt="Most people think machines will take their jobs", max_tokens=50)
response['choices'][0]['text']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11392405063291139, 'fraction_numerical': 0.014384349827387802, 'mean_word_length': 8.502732240437158, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '135270', 'license': 'MIT', 'n_tokens_mistral': 624, 'n_tokens_neox': 594, 'n_words': 111, 'path': 'eda.ipynb', 'repo_name': 'idriskuti2/hate-speech-and-offensive-language', 'size': '1776'}