<jupyter_start><jupyter_text>This notebook performs initial preprocessing of three sets of data 1. Annotated data 1 (ca.1000 tweets) 2.Annotated data 2 (ca.500 tweets) 3. Non annotated data (ca.12000 tweets). Schizophrenia data is saved down in the data/dataOut/schiz/ path. The annotated data has the class label 1 removed to convert this into a binary task.<jupyter_code>import pandas as pd
import numpy as np<jupyter_output><empty_output><jupyter_text>Twitter Scizhophrenia stigma<jupyter_code>'''
import csv file and return dataframe, can
pass column if want to return series
'''
def getFile(fileName, column=None):
    socialDf = pd.read_csv(fileName, encoding='utf-8')
    return socialDf if column==None else socialDf[column]


'''
clean the schizophrenia data. Remove class labels 1 to
return convert into binary classification task. Turn 
label 2 into 1
'''
def cleanScizAnnTwitter(annTwitter):

    print('initially', annTwitter['Classification'].value_counts())
    annTwitter = annTwitter.dropna()
    annTwitter = annTwitter[annTwitter['Classification']!='o']
    annTwitter = annTwitter.astype({"Classification": int})
    annTwitter = annTwitter[annTwitter['Classification']!=1]
    
    annTwitter = annTwitter.replace(2, 1)
    
    print('initially', annTwitter['Classification'].value_counts())
    
    
    return annTwitter<jupyter_output><empty_output><jupyter_text>Generate files for baseline model<jupyter_code>fileName = 'data/baseline/dataIn/annFinalSchiz_1.csv'
socialDf = getFile(fileName)
print len(socialDf)
tweets = cleanScizAnnTwitter(socialDf)
print len(tweets), 'grg'
tweets.to_csv('data/baseline/dataOut/annSchiz1.csv', encoding='utf-8')
socialDf['Classification'].value_counts()
tweets['Classification'].value_counts()
fileName = 'data/baseline/dataIn/annFinalSchiz_2.csv'
socialDf = getFile(fileName)
tweets = cleanScizAnnTwitter(socialDf)
tweets.to_csv('data/baseline/dataOut/annSchiz2.csv', encoding='utf-8')<jupyter_output>('initially', 0.0    167
2.0    163
1.0    103
Name: Classification, dtype: int64)
('initially', 0    167
1    163
Name: Classification, dtype: int64)<jupyter_text>Generate files for Schizophrenia<jupyter_code>fileName = 'data/dataIn/schiz/nonAnnFinalSchiz.csv'
socialDf = getFile(fileName)
socialDf.to_csv('data/dataOut/schiz/nonAnnFinalSchiz.csv', encoding='utf-8')
fileName = 'data/dataIn/schiz/annFinalSchiz_1.csv'
socialDf = getFile(fileName)
tweets = cleanScizAnnTwitter(socialDf)
tweets.to_csv('data/dataOut/schiz/annFinalSchiz_1.csv', encoding='utf-8')
tweets['Classification'].value_counts()
fileName = 'data/dataIn/schiz/annFinalSchiz_2.csv'
socialDf = getFile(fileName)
tweets = cleanScizAnnTwitter(socialDf)
tweets.to_csv('data/dataOut/schiz/annFinalSchiz_2.csv', encoding='utf-8')<jupyter_output><empty_output><jupyter_text>generate files for Stigma Data - Not used in final project<jupyter_code>fileName = 'data/dataIn/stigma/nonAnnFinalStig.csv'
socialDf = getFile(fileName)
socialDf.to_csv('data/dataOut/stigma/nonAnnFinalStig.csv', encoding='utf-8')
fileName = 'data/dataIn/stigma/annFinalStig.csv'
socialDf = getFile(fileName)
stigTwitter = socialDf.dropna()
stigTwitter = stigTwitter[socialDf['Classification']!='Neutral']
stigTwitter = stigTwitter[socialDf['Classification']!='Unclear']
stigTwitter = stigTwitter.replace('Anti-stigma', 0)
stigTwitter = stigTwitter.replace('Stigma', 1)
stigTwitter.to_csv('data/dataOut/stigma/annFinalStig.csv', encoding='utf-8')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12226697353279632, 'fraction_numerical': 0.019850402761795168, 'mean_word_length': 6.8487584650112865, 'pattern_counts': {'":': 1, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '272291', 'license': 'MIT', 'n_tokens_mistral': 1278, 'n_tokens_neox': 1198, 'n_words': 229, 'path': 'notebooks/01 preprocessing.ipynb', 'repo_name': 'gregoryverghese/schizophrenia-twitter', 'size': '3476'}