<jupyter_start><jupyter_text>Autoselecting genes related to cancer<jupyter_code>with open('stopwords.txt', 'r') as f:
    stops = f.readlines()
sentences =pd.read_csv('genesdf')
sentences=sentences.drop(['Unnamed: 0'],axis=1)
sentences.head()
import re

REPLACE_BY_SPACE_RE = re.compile('[/(){}\[\]\|@,;]')
STOPWORDS = stops
def clean_text(text):
    """
        text: a string
        
        return: modified initial string
    """
    text = text.lower() # lowercase text
    text = REPLACE_BY_SPACE_RE.sub(' ', text) # replace REPLACE_BY_SPACE_RE symbols by space in text. substitute the matched string in REPLACE_BY_SPACE_RE with space.
    text = ' '.join(word for word in text.split() if word not in STOPWORDS) # remove stopwords from text
    return text
sentences['blurbtext']=sentences['blurb'].apply(lambda x: str(x).lower())
sentences['blurbtext']=sentences['blurbtext'].apply(lambda x: clean_text(x))
#sentences['blurbsplit']=sentences['blurbtext'].apply(lambda x: str(x).split('.')[0])
len(sentences['blurbtext'][0])
test="I don't have myeloma."
pd.Series(test).str.contains('tumor|\w*oma\W')
sentences['predict'] = sentences['blurbtext'].str.contains('tumor|cancer|leukemia|\w*oma\W')
len(sentences)
len(sentences[sentences['predict']==1])
lenblurb=[len(x.split()) for x in sentences['blurbtext']]
maxlen=max(lenblurb)
print('max sequence')
maxlen
#make words into one-hot-vectors

def make_vector(word,vocab):
    lenvocab=len(vocab)
    blurbV=[]

    vector=np.zeros(lenvocab)
    vector[vocab.index(word)]=1

    return vector

    
#for i in blurbs.columns:
    #blurbs[i]=blurbs[i].apply(lambda x: make_vector('tumor',vocab) if x==x else np.zeros(len(vocab)))
from keras.preprocessing.text import Tokenizer
# The maximum number of words to be used. (most frequent)
MAX_NB_WORDS = 2000
# Max number of words in each complaint.
MAX_SEQUENCE_LENGTH = 298
# This is fixed.
EMBEDDING_DIM = 1000
tokenizer = Tokenizer(num_words=MAX_NB_WORDS, filters='!"#$%&()*+,-./:;<=>?@[\]^_`{|}~', lower=True)
tokenizer.fit_on_texts(sentences['blurbtext'].values)
word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
from keras.preprocessing.sequence import pad_sequences
X = tokenizer.texts_to_sequences(sentences['blurbtext'].values)
X = pad_sequences(X, maxlen=MAX_SEQUENCE_LENGTH)
print('Shape of data tensor:', X.shape)
X
#X=blurbs
y=np.array(sentences['predict'])
#you can use get_dummies if y is not binary
#print('Shape of data tensor:', X.shape)
#print('Shape of label tensor:', y.shape)
from sklearn.model_selection import train_test_split
X_train, X_test, Y_train, Y_test = train_test_split(X,y, test_size = 0.10, random_state = 42)
print(X_train.shape,Y_train.shape)
print(X_test.shape,Y_test.shape)
from random import randint
from numpy import array
from numpy import argmax
from pandas import DataFrame

from keras.models import Sequential
from keras.layers import LSTM
from keras.layers import Flatten
from keras.layers import Dropout
from keras.layers import Dense
from keras.layers import TimeDistributed
from keras.layers import RepeatVector
from keras.layers import Embedding
from keras.layers import Activation
from keras.layers.core import Dense, SpatialDropout1D
from keras.callbacks import EarlyStopping
X_train.shape[1]
epochs = 5
batch_size = 16
EMBEDDING_DIM=1000
model = Sequential()
model.add(Embedding(MAX_NB_WORDS,EMBEDDING_DIM, input_length=X.shape[1]))
model.add(LSTM(1500, dropout=0.2))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])


history = model.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size,validation_split=0.1,callbacks=[EarlyStopping(monitor='val_loss', patience=3, min_delta=0.0001)])
accr = model.evaluate(X_test,Y_test)
print('Test set\n  Loss: {:0.3f}\n  Accuracy: {:0.3f}'.format(accr[0],accr[1]))
plt.title('Loss')
plt.plot(history.history['loss'], label='train')
plt.plot(history.history['val_loss'], label='test')
plt.legend()
plt.show();
plt.title('Accuracy')
plt.plot(history.history['acc'], label='train')
plt.plot(history.history['val_acc'], label='test')
plt.legend()
plt.show();<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1486873508353222, 'fraction_numerical': 0.012649164677804296, 'mean_word_length': 6.551351351351351, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '551721', 'license': 'MIT', 'n_tokens_mistral': 1563, 'n_tokens_neox': 1486, 'n_words': 321, 'path': 'LSTM-Copy1.ipynb', 'repo_name': 'maymag/ds_projects', 'size': '4190'}