<jupyter_start><jupyter_text>socialCARDS data preparation This script loads and combines the **convinced** (0) and **contrarian** (1) posts sourced from **facebook and  twitter**, and splits it into **training, validation and testing** data sets.  To get some measure of external validity, the testing data set is a "pure" held out data set that only contains text from "unseen" sources, i.e.  twitter and facebook accounts that the classifier was not trained on.  Packages & functions<jupyter_code>import os
import random
import json
import pandas as pd
from sklearn.model_selection import train_test_split
import nltk
def train_valid_test_split(data, account_variable,
                           random_test_sample_size = .15, 
                           valid_sample_size = .25, 
                           random_seed = 1, random_state = 1, 
                           shuffle = True):
    """Split the data into training, validation and testing data set, with the testing 
    data being completely held-out data by unseen accounts"""
    
    # Set a random seed
    random.seed(random_seed)
    
    # Extract unique usernames for each side
    contrarian_accounts = data[data["label"]==1][account_variable].unique()
    convinced_accounts = data[data["label"]==0][account_variable].unique()
    
    # Create a randomn sample of users for the testing set for each side
    samplesize_contrarian = int(round(random_test_sample_size*len(contrarian_accounts),0))
    test_contrarian = random.sample(list(contrarian_accounts), samplesize_contrarian)
    samplesize_convinced = int(round(random_test_sample_size*len(convinced_accounts),0))
    test_convinced = random.sample(list(convinced_accounts), samplesize_convinced)
    
    # Separate the testing dataset
    trainvalid = pd.concat([data[~data[account_variable].isin(test_contrarian + test_convinced)]])
    test = pd.concat([data[data[account_variable].isin(test_contrarian + test_convinced)]])
    
    # Split training data into training and validation dataset
    train, valid = train_test_split(trainvalid, test_size=valid_sample_size, 
                                    random_state=random_state, shuffle=shuffle)
    print("Training data set with {} posts created.".format(len(train)))
    print("Validation data set with {} posts created.".format(len(valid)))    
    print("Testing data set with {} posts created.".format(len(test)))    

    return train, valid, test

def load_tweets(folderpath, label, verbose = True):
    """This function loads twitter json files into a pandas dataframe and assigns them a
    numeric label."""
    tweets, username, name, date, time = [[] for _ in range(5)]
    for i, file in enumerate(os.listdir(folderpath)):
        if verbose == True:
            print(i, file)
        for line in open('/'.join([folderpath, file]), 'r'):
            tweets.append(json.loads(line)['tweet'])
            username.append(json.loads(line)['username'])
            name.append(json.loads(line)['name'])
            date.append(json.loads(line)['date'])
            time.append(json.loads(line)['time'])
    df = pd.DataFrame(list(zip(tweets, [label]*len(tweets), username, name, date, time)),
                    columns = ['text', 'label', 'username', 'name', 'date', 'time'])
    if verbose == True:
        print("\n")
    return df

def load_facebook(folderpath, label, verbose = True):
    """This function loads facebook json files into a pandas dataframe and assigns them a
    numeric label."""
    text, handle, ID, name, pageAdminTopCountry, pageCategory, date, postUrl, link = [[] for _ in range(9)]
    for i, file in enumerate(os.listdir(folderpath)):
        if verbose == True:
            print(i, file)
        posts = json.load(open('/'.join([folderpath, file])))['posts']
        for post in posts:
            text.append(post.get('message'))
            handle.append(post['account'].get('handle'))
            ID.append(post['account'].get('id'))
            name.append(post['account'].get('name'))
            pageAdminTopCountry.append(post['account'].get('pageAdminTopCountry'))
            pageCategory.append(post['account'].get('pageCategory'))
            date.append(post.get('date'))
            postUrl.append(post.get('postUrl'))
            link.append(post.get('link'))     
    df = pd.DataFrame(list(zip(text, [label]*len(text), handle, ID, name, pageAdminTopCountry, pageCategory, date, postUrl, link)),
                    columns = ['text', 'label', 'handle', 'ID', 'name', 'pageAdminTopCountry', 'pageCategory', 'date', 'postUrl', 'link'])
    df = df.dropna(subset=['text'])
    if verbose == True:
        print("\n")
    return df<jupyter_output><empty_output><jupyter_text>Load data  Twitter<jupyter_code>print('---Load the contrarian tweets---\n')
twitter_contrarian = load_tweets('../data/twitter/contrarian', label = 1, verbose = False)
print(len(twitter_contrarian), "contrarian tweets loaded.")

print('\n\n---Load the convinced tweets---\n')
twitter_convinced = load_tweets('../data/twitter/convinced', label = 0, verbose = False)
print(len(twitter_convinced), "convinced tweets loaded.\n\n")

print('---Split the data (test data: unseen twitter handles)---\n')
twitter = pd.concat([twitter_contrarian, twitter_convinced])
twitter['type'] = "twitter"
twitter_train, twitter_valid, twitter_test = train_valid_test_split(twitter, "username") # Here the held out data is unseen twitter accounts.<jupyter_output>---Load the contrarian tweets---

1966699 contrarian tweets loaded.


---Load the convinced tweets---

1573621 convinced tweets loaded.


---Split the data (test data: unseen twitter handles)---

Training data set with 2431705 posts created.
Validation data set with 810569 posts created.
Testing data set with 298046 posts created.<jupyter_text>Facebook<jupyter_code>print('---Load the contrarian facebook posts---\n')
facebook_contrarian = load_facebook('../data/facebook/contrarian', label = 1, verbose = False)
print(len(facebook_contrarian), "contrarian facebook posts loaded.")
# 449199 contrarian facebook posts loaded.

print('\n\n---Load the convinced facebook posts---\n')
facebook_convinced = load_facebook('../data/facebook/convinced', label = 0, verbose = False)
print(len(facebook_convinced), "convinced facebook posts loaded.\n\n")

print('---Split the data (test data: unseen facebook handles)---\n')
facebook = pd.concat([facebook_contrarian, facebook_convinced])
facebook['type'] = "facebook"
facebook_train, facebook_valid, facebook_test = train_valid_test_split(facebook, "handle") # Here the held out data is unseen facebook accounts.<jupyter_output>---Load the contrarian facebook posts---

425551 contrarian facebook posts loaded.


---Load the convinced facebook posts---

527951 convinced facebook posts loaded.


---Split the data (test data: unseen facebook handles)---

Training data set with 663504 posts created.
Validation data set with 221168 posts created.
Testing data set with 68830 posts created.<jupyter_text>Combine & export data<jupyter_code>train = pd.concat([twitter_train[['text', 'label', 'type']],
                  facebook_train[['text', 'label', 'type']]]).reset_index(drop = True)
train
valid = pd.concat([twitter_valid[['text', 'label', 'type']],
                  facebook_valid[['text', 'label', 'type']]]).reset_index(drop = True)
valid
test = pd.concat([twitter_test[['text', 'label', 'type']],
                  facebook_test[['text', 'label', 'type']]]).reset_index(drop = True)
test
train[['type', 'label']][train.label == 1].type.value_counts(normalize=True)
train[['type', 'label']][train.label == 0].type.value_counts(normalize=True)
train.label.value_counts(normalize=True)
# Save the data sets
train.to_csv('../data/socialCARDStrain.csv', index = False, encoding = 'utf-8')
valid.to_csv('../data/socialCARDSvalid.csv', index = False, encoding = 'utf-8')
test.to_csv('../data/socialCARDStest.csv', index = False, encoding = 'utf-8')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13522997869407194, 'fraction_numerical': 0.011404937962150646, 'mean_word_length': 3.895705521472393, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '295621', 'license': 'Apache-2.0', 'n_tokens_mistral': 2442, 'n_tokens_neox': 2303, 'n_words': 626, 'path': 'code/.ipynb_checkpoints/socialCARDS_data_prep-checkpoint.ipynb', 'repo_name': 'MirjamOdile/babyCARDS', 'size': '7979'}