<jupyter_start><jupyter_text>Code for cleanning and preparing the entry surveys data<jupyter_code>from __future__ import absolute_import, division, print_function
import datetime
import time
import os

import pandas as pd
import sys
sys.path.insert(0, '../../src/data/')

from config import *<jupyter_output><empty_output><jupyter_text>Config<jupyter_code># Matplotlib for additional customization
from matplotlib import pyplot as plt
%matplotlib inline

# Seaborn for plotting and styling
import seaborn as sns<jupyter_output>2019-01-18 13:20:44,777 - DEBUG - backend module://ipykernel.pylab.backend_inline version unknown<jupyter_text>read<jupyter_code>participants_entry_survey_data_anon = pd.read_hdf(surveys_anon_store_path, 'entry/participants_entry_survey_data_anon')
participants_entry_survey_data_anon.head()<jupyter_output><empty_output><jupyter_text>Rename columns<jupyter_code>columns_dict = {
    'member':'member',
    'Q3.3':'race',
    'Q3.3_7_TEXT':'race_other',
    'Q3.2':'age',
    'Q3.1':'gender',
    'Q3.4':'citizneships',
    'Q5.3':'is_cofounder',
    'Q5.5_1_TEXT':'title',
    'Q5.6':'time_in_startup',
    'Q6.1':'HHH_type',
    'Q6.2':'experience',
    'Q7.2_1':'TIPI_1',
    'Q7.2_2':'TIPI_2',
    'Q7.2_3':'TIPI_3',
    'Q7.2_4':'TIPI_4',
    'Q7.2_5':'TIPI_5',
    'Q7.2_6':'TIPI_6',
    'Q7.2_7':'TIPI_7',
    'Q7.2_8':'TIPI_8',
    'Q7.2_9':'TIPI_9',
    'Q7.2_10':'TIPI_10',
}
participants_entry_survey_data_clean = participants_entry_survey_data_anon.rename(columns=columns_dict)
participants_entry_survey_data_clean.head(5)<jupyter_output><empty_output><jupyter_text>Translate values  Big 5 TIPI  Transalte text to numbers<jupyter_code>participants_entry_survey_data_clean.TIPI_1.unique()
def TIPI_Translation(answer):
    """
    A function to translate the string answers from the TIPI survey to numbers
    """
    if answer == 'Agree strongly':
        return(7.0)
    if answer == 'Agree moderately':
        return(6.0)
    if answer == 'Agree a little':
        return(5.0)
    if answer == 'Neither agree nor disagree':
        return(4.0)
    if answer == 'Disagree a little':
        return(3.0)
    if answer == 'Disagree moderately':
        return(2.0)
    if answer == 'Disagree strongly ': #extra space....
        return(1.0)
    if answer == 'Disagree strongly':
        return(1.0)
    return answer
#Apply translation functions
for i in range(1,11):
    participants_entry_survey_data_clean['TIPI_{}'.format(i)] = participants_entry_survey_data_clean['TIPI_{}'.format(i)].apply(TIPI_Translation)<jupyter_output><empty_output><jupyter_text>Transalte answers to personalityTIPI scale scoring (“R” denotes reverse-scored items):* Extraversion: 1, 6R* Agreeableness: 2R, 7* Conscientiousness; 3, 8R* Emotional Stability: 4R, 9* Openness to Experiences: 5, 10RScoring the TIPI:1. Recode the reverse-scored items (i.e., recode a 7 with a 1, a 6 with a 2, a 5 with a 3, etc.). The reverse scored items are 2, 4, 6, 8, & 10.2. Take the AVERAGE of the two items (the standard item and the recoded reverse-scored item) that make up each scale.Example using the Extraversion scale: A participant has scores of 5 on item 1 (Extraverted, enthusiastic) and and 2 on item 6 (Reserved, quiet). First, recode the reverse-scored item (i.e., item 6), replacing the 2 with a 6. Second, take the average of the score for item 1 and the (recoded) score for item 6. So the TIPI Extraversion scale score would be: (5 + 6)/2 = 5.5<jupyter_code>for i in range(2,11,2):
        participants_entry_survey_data_clean['TIPI_{}R'.format(i)] = 8 - participants_entry_survey_data_clean['TIPI_{}'.format(i)]
df = participants_entry_survey_data_clean
participants_entry_survey_data_clean['TIPI_extraversion']        = (df['TIPI_1']+df['TIPI_6R'])/2
participants_entry_survey_data_clean['TIPI_agreeableness']       = (df['TIPI_2R']+df['TIPI_7'])/2
participants_entry_survey_data_clean['TIPI_conscientiousness']   = (df['TIPI_3']+df['TIPI_8R'])/2
participants_entry_survey_data_clean['TIPI_emotional_stability'] = (df['TIPI_4R']+df['TIPI_9'])/2
participants_entry_survey_data_clean['TIPI_openness']            = (df['TIPI_5']+df['TIPI_10R'])/2

for i in range(1,11):
    del participants_entry_survey_data_clean['TIPI_{}'.format(i)]
    
for i in range(2,11,2):
    del participants_entry_survey_data_clean['TIPI_{}R'.format(i)]
participants_entry_survey_data_clean.head()<jupyter_output><empty_output><jupyter_text>is CEO?<jupyter_code>participants_entry_survey_data_clean.title.value_counts()
#CEO
#Ceo
#participants_entry_survey_data_clean['is_ceo'] = \
#    participants_entry_survey_data_clean.title.str.contains('ceo',case=False)\
#    .fillna(False)


participants_entry_survey_data_clean['is_ceo'] = 0
cond = (participants_entry_survey_data_clean.title.fillna("").str.contains("ceo",case=False))
participants_entry_survey_data_clean.loc[cond,'is_ceo'] = 1
participants_entry_survey_data_clean.query('is_ceo == 1')[['title','is_ceo']]
participants_entry_survey_data_clean[['member','title','is_ceo']].query('is_ceo')
participants_entry_survey_data_clean.head()<jupyter_output><empty_output><jupyter_text>Is co-founder<jupyter_code># Set for 'Yes and maybe'
participants_entry_survey_data_clean['is_cofounder_temp'] = 0
cond = (participants_entry_survey_data_clean.is_cofounder != 'No')
participants_entry_survey_data_clean.loc[cond,'is_cofounder_temp'] = 1
print(len(participants_entry_survey_data_clean.query('is_cofounder_temp == 1')))

# Manually fix
cond = (participants_entry_survey_data_clean.member == 'XLIPIHEOIT')
participants_entry_survey_data_clean.loc[cond,'is_cofounder_temp'] = 0
print(len(participants_entry_survey_data_clean.query('is_cofounder_temp == 1')))
participants_entry_survey_data_clean['is_cofounder'] = participants_entry_survey_data_clean['is_cofounder_temp']
del participants_entry_survey_data_clean['is_cofounder_temp']<jupyter_output><empty_output><jupyter_text>HHH<jupyter_code>participants_entry_survey_data_clean.HHH_type.value_counts()
def HHH_Translation(answer):
    """
    """
    if answer == 'Hacker (you can solve any technical problem and make anything work)':
        return('Hacker')
    if answer == 'Hustler (you are the one who closes deals and brings back the money)':
        return('Hustler')
    if answer == 'Hipster (you are a creative design genius who makes the user experience awesome)':
        return('Hispster')
    return answer
participants_entry_survey_data_clean['HHH_type'] = participants_entry_survey_data_clean['HHH_type'].apply(HHH_Translation)  
participants_entry_survey_data_clean.HHH_type.value_counts()
participants_entry_survey_data_clean.head()<jupyter_output><empty_output><jupyter_text>Gender<jupyter_code>def gender_Translation(answer):
    """
    """
    if answer == 'Male':
        return('M')
    if answer == 'Female':
        return('F')
    else:
        return('U')
    return answer

participants_entry_survey_data_clean['gender'] = participants_entry_survey_data_clean['gender'].apply(gender_Translation)  
#participants_entry_survey_data_clean.query('gender == "U"')
#participants_entry_survey_data_clean.loc['UAXR5EMOI2','gender']='M'

participants_entry_survey_data_clean.loc[participants_entry_survey_data_clean.member =='UAXR5EMOI2','gender']='M'
participants_entry_survey_data_clean.gender.value_counts()<jupyter_output><empty_output><jupyter_text>tests<jupyter_code>participants_entry_survey_data_clean.dtypes
participants_entry_survey_data_clean.HHH_type.value_counts()
participants_entry_survey_data_clean.TIPI_extraversion.hist()
participants_entry_survey_data_clean.TIPI_agreeableness.hist()
participants_entry_survey_data_clean.TIPI_openness.hist()<jupyter_output><empty_output><jupyter_text>Store<jupyter_code>participants_entry_survey_data_clean.set_index('member', inplace=True)
with pd.HDFStore(surveys_clean_store_path) as store:
    store.put('entry/participants_entry_survey_data_clean', participants_entry_survey_data_clean, format='table')<jupyter_output><empty_output><jupyter_text>Sainty check<jupyter_code>members = pd.read_hdf(analysis_store_path, 'metadata/members')
members.head()
a = participants_entry_survey_data_clean.join(members)
a.query('is_ceo == 1')[['company']].sort_values('company')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12314019595984033, 'fraction_numerical': 0.022257167049715736, 'mean_word_length': 6.792648444863336, 'pattern_counts': {'":': 0, '<': 52, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '359866', 'license': 'MIT', 'n_tokens_mistral': 3205, 'n_tokens_neox': 3030, 'n_words': 483, 'path': 'notebooks/10-data-prep/01.20-ol-entry.ipynb', 'repo_name': 'HumanDynamics/rhythm-public-analysis-deltav18-anon', 'size': '8267'}