<jupyter_start><jupyter_text>This notebook contains the code that uses spaceyMoji to tokenize our text. This gets around the nltk problem<jupyter_code>import spacy
import numpy as np
from spacymoji import Emoji
import pandas as pd
import emoji
import nltk
from gensim.models import Word2Vec
annSchiz1 = pd.read_csv('data/baseline/dataOut/annSchiz1.csv', encoding='utf-8')
annSchiz2 = pd.read_csv('data/baseline/dataOut/annSchiz2.csv', encoding='utf-8')
nonAnnSchizFile = pd.read_csv('data/dataIn/schiz/nonAnnFinalSchiz.csv', encoding='utf-8')
nlp = spacy.load('en')
emoji = Emoji(nlp)
nlp.add_pipe(emoji, first=True)
def checkEmoji(sent, check=False):
    doc = nlp(sent)
    if doc._.has_emoji==check:
        sent = np.nan
    return sent

def getEmojiDf(df, check=False):
    dfNew = df.copy()
    dfNew['Tweet'] = dfNew['Tweet'].apply(lambda x: checkEmoji(x, check))
    dfNew = dfNew.dropna()
    return dfNew

def removeEmojis(sent):
    doc = nlp(sent)
    tokens = [token.text for token in doc if not token._.is_emoji]
    sent = ' '.join(tokens)
    return sent<jupyter_output><empty_output><jupyter_text>Baseline<jupyter_code>emojiDf = getEmojiDf(annSchiz1)
emojiDf.to_csv('data/baseline/emoji/emSchiz1Em.csv')
emojiDf['Classification'].value_counts()
noEmojiDf = emojiDf.copy()
noEmojiDf['Tweet'] = noEmojiDf['Tweet'].apply(lambda x: removeEmojis(x))
noEmojiDf.to_csv('data/baseline/emoji/Schiz1Em.csv')
emojiDf = getEmojiDf(annSchiz2)
emojiDf.to_csv('data/baseline/emoji/emSchiz2Em.csv')
emojiDf['Classification'].value_counts()
noEmojiDf = emojiDf.copy()
noEmojiDf['Tweet'] = noEmojiDf['Tweet'].apply(lambda x: removeEmojis(x))
noEmojiDf.to_csv('data/baseline/emoji/Schiz2Em.csv')<jupyter_output><empty_output><jupyter_text>Embeddings<jupyter_code>min_count = [2, 3, 4, 5]
window = [2, 3 , 4, 5]
size = [25, 50, 100, 200]
sample=6e-5
alpha=0.05
min_alpha=0.0007 
negative=20
sgTrain = 1
cbowTrain = 0
emArgs = [min_count[0], window[0], size[3], sample, alpha, min_alpha, negative]
def tokenizeTweet(sent):
    doc = nlp(sent)
    tokens= [t.text.strip() for t in doc]
    return tokens
nonAnnSchiz = nonAnnSchizFile[['Tweet','true_false']]
#emojiDf = getEmojiDf(nonAnnSchiz)
emojitokens = nonAnnSchiz['Tweet'].apply(lambda x: tokenizeTweet(x))
emojitokens.to_csv('data/dataOut/schiz/emoji/nonAnnFinalSchizEmoji.csv')
charTokens = nonAnnSchiz['Tweet'].apply(lambda x: list(x))
charTokens.to_csv('data/dataOut/schiz/emoji/nonAnnFinalSchizEmojiChar.csv')
def getEmoticonText(tokens):

    emojiLines = list(map(lambda x: any(t._.is_emoji for t in x), tokens))
    tokens = np.array(tokens)
    emojiLines = np.array(emojiLines)
  
    tokens = tokens[emojiLines]
 
    return tokens
emojiLst = []
y = emojiDf['Tweet'].tolist()
for t in y:
    doc = nlp(t)
    for token in doc:
        if token._.is_emoji:
            emojiLst.append(token.text)
## Train and Test Data 
emojitokens = annSchiz2['Tweet'].apply(lambda x: tokenizeTweet(x))
emojiDf = getEmojiDf(annSchiz2)
len(emojiDf)
z = emojiDf['Tweet'].apply(lambda x: nltk.word_tokenize(x))
x = z.tolist()
emojitokens.to_csv('data/dataOut/schiz/emoji/annSchiz2.csv')
emojitokens = annSchiz2['Tweet'].apply(lambda x: list(x))
emojitokens.to_csv('data/dataOut/schiz/emoji/annSchizChar2.csv')<jupyter_output>/usr/local/lib/python3.7/site-packages/ipykernel_launcher.py:2: FutureWarning: The signature of `Series.to_csv` was aligned to that of `DataFrame.to_csv`, and argument 'header' will change its default value from False to True: please pass an explicit value to suppress this warning.
{'fraction_non_alphanumeric': 0.13750352211890673, 'fraction_numerical': 0.016060862214708368, 'mean_word_length': 5.501831501831502, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '667360', 'license': 'MIT', 'n_tokens_mistral': 1439, 'n_tokens_neox': 1430, 'n_words': 263, 'path': 'notebooks/09 spacymoji.ipynb', 'repo_name': 'gregoryverghese/schizophrenia-twitter', 'size': '3549'}