<jupyter_start><jupyter_text>```python define a matrix of relationships ... realized this is harder than it looks! 0 - not related 1 - father 2 - mother 3 - sibling 4 - child 5 - otherfam = pd.DataFrame(np.zeros([df.shape[0],df.shape[0]]), index=df.index, columns=df.index)for i, char in enumerate(fam.index):    curr_fam = df.loc[char, 'Family']    if curr_fam is np.nan or curr_fam is None:        continue         otherwise, let's parse their family tree    common_fam = [filter(lambda x: x in curr_fam, sublist) for sublist in fam.index]    print(curr_fam)    print(common_fam)    break```<jupyter_code># define some useful subfunctions

# given a dataframe/regex phrase, this counts the number of times the regex appears
def count_words(s, phrase):
    if s is np.nan:
        return 0
    else:
        return len(re.findall(phrase,s, re.IGNORECASE))

# this calls the above function and adds the data to the given dataframe
def add_data(df_data, column_name, phrase, txt_col=None):
    if txt_col is None:
        txt_col = ['Background','Season 1','Season 2', 'Season 3', 'Season 4']
        
    df_new = df[txt_col].applymap(lambda x: count_words(x,phrase))
    df_data[column_name] = df_new[txt_col].sum(axis=1)
    return df_data

def add_allegiance(df, df_has_allegiance):
    
    houses = ['House Lannister', 'House Stark', 'House Targaryen', 'House Frey', 
              'House Greyjoy', "Night's Watch", 'House Martell', 'Free Folk', 'House Forrester',
              'House Bolton', 'House Baratheon', 'Kingsguard']
    
    for h in houses:
        tmp = np.zeros(df.shape[0])
        val = df_has_allegiance['Allegiance'].values
        for i in range(df.shape[0]):
            if val[i] is np.nan:
                continue
            elif h in val[i]:
                tmp[i] = 1.0
        df[h] = tmp
        
    return df
def extract_data(df, s=3):
    # given the dataframe and the season, extract some features
    # only look at background // season 1
    
    # by default, extract data up to and including season 3, i.e. s=3
    txt_col = ['Background','Season 1','Season 2','Season 3','Season 4',' Season 5']
    
    txt_col = txt_col[0:(s+1)]
    
    # initialize the dataframe
    idxData = np.ones(df.shape[0], dtype=bool)
    df_data = df.loc[idxData, ['Status','Season(s)']]

    # start adding data based on counting the frequency of words

    # count the number of hyphens as a surrogate as the number of in-show family member
    # in the data, the family members appear as "sister - Sansa Stark, father - <NAME>ark," .. etc
    df_data = add_data(df_data, 'family_members',' - ', txt_col = ['Family'])

    #df_data = add_allegiance(df_data, df)
    
    # number of seasons the character appeared in
    df_data = add_data(df_data, 'number_of_seasons','[1-' + str(s) + ']', txt_col = ['Season(s)'])

    # various word "types" across all the data
    df_data = add_data(df_data, 'violent_words','(knight|warrior|sword|axe|spear|kill|murder|fight|assassinate)')
    df_data = add_data(df_data, 'sexy_words','(sex|naked|love|slept|nude|kiss)')
    df_data = add_data(df_data, 'fun_words','(beer|wine|glass|drunk|inebriate)')
    df_data = add_data(df_data, 'number_of_words','\w+')

    # remove people who died before current season
    idxRemove = np.zeros(df_data.shape[0],dtype=bool)
    for i, val in enumerate(df_data['Status'].values):
        char = df_data.index[i]
        if val is None or val is np.nan:
            continue
        if ('Dead' in val) | ('Deceased' in val):
            if len(re.findall('[' + str(s+1) + '-5]', df_data.loc[char, 'Season(s)'])) > 0:
                continue
            else:
                idxRemove[i] = True

    df_data = df_data.loc[~idxRemove]

    df_data.drop('Season(s)',axis=1,inplace=True)
    return df_data
    
df_data = extract_data(df, s=4)
df_data.head(n=2)
# prep data
X = df_data.values[:,1:df_data.shape[1]]

# alive until proven dead !
y = np.zeros(df_data.shape[0])
for i, val in enumerate(df_data['Status'].values):
    if val is None or val is np.nan:
        continue
    if 'Dead' in val:
        y[i] = 1.0
    elif 'Deceased' in val:
        y[i] = 1.0
        
# workaround cross_val_predict returning 0s/1s - calling this makes it return probabilities
class proba_logreg(LogisticRegression):
    def predict(self, X):
        return LogisticRegression.predict_proba(self, X)

# cross-validation performance
mdl = "logreg"
model = LogisticRegression(fit_intercept=True)
estimator = Pipeline([("imputer", Imputer(missing_values='NaN',
                                      strategy="mean",
                                      axis=0)),
                  ("scaler", StandardScaler()),
                  ("lr", model)])
scores = cross_val_score(estimator, X, y, scoring='roc_auc',cv=5)
print('{:10s} {:5g} [{:5g}, {:5g}]'.format("lr", np.mean(scores), np.min(scores), np.max(scores) ))

# plot a roc curve by getting cross-validation predictions
predicted = cross_val_predict(proba_logreg(), X, y, cv=10)
predicted = predicted[:,1]

plt.figure(figsize=[9,9])
ax = plt.gca()
fpr, tpr, thresholds = metrics.roc_curve(y, predicted, pos_label=1)
plt.plot(fpr, tpr, 'bo-',lw=2,markersize=12,
        label=mdl + ' ' + '%0.3f' % metrics.auc(fpr, tpr))
plt.xlabel('False positive rate',fontsize=14)
plt.ylabel('True positive rate',fontsize=14)
plt.legend(loc='lower right')
plt.title('Mortality prediction',fontsize=16)
plt.show()
sort_indices = np.argsort(predicted)

for i in range(20):
    idx = sort_indices[-i-1]
    print('{:20s}: {:0.5f} (outcome: {})'.format(df_data.index[idx],
                                        predicted[idx],
                                       y[idx]))<jupyter_output>Nymeria_Sand        : 0.44362 (outcome: 0.0)
Walder_Frey         : 0.44311 (outcome: 0.0)
Lancel              : 0.43200 (outcome: 0.0)
Shireen_Baratheon   : 0.40623 (outcome: 1.0)
Obara_Sand          : 0.40508 (outcome: 0.0)
Steffon_Baratheon   : 0.40018 (outcome: 1.0)
Tyene_Sand          : 0.39454 (outcome: 0.0)
Podrick_Payne       : 0.34880 (outcome: 0.0)
Willas_Tyrell       : 0.33750 (outcome: 0.0)
Rickon_Stark        : 0.33383 (outcome: 0.0)
Edmure_Tully        : 0.31655 (outcome: 0.0)
Robin_Arryn         : 0.31235 (outcome: 0.0)
Trystane_Martell    : 0.31096 (outcome: 0.0)
Daario_Naharis      : 0.31065 (outcome: 0.0)
Benjen_Stark        : 0.30714 (outcome: 0.0)
Walda_Bolton        : 0.30200 (outcome: 0.0)
Rickard_Stark       : 0.29922 (outcome: 1.0)
Lothar_Frey         : 0.29087 (outcome: 0.0)
Elia_Martell        : 0.28288 (outcome: 1.0)
Sansa_Stark         : 0.28271 (outcome: 0.0)<jupyter_text>TF-IDFLet's try TF-IDF of the same data !<jupyter_code>def extract_data_text(df, s=3):
    # given the dataframe and the season, extract some features
    # only look at background // season 1
    
    # by default, extract data up to and including season 3, i.e. s=3
    txt_col = ['Background','Season 1','Season 2','Season 3','Season 4', 'Season 5']
    
    txt_col = txt_col[0:(s+1)]
    
    # initialize the dataframe
    idxData = np.ones(df.shape[0], dtype=bool)
    df_data = df.loc[idxData, ['Status','Season(s)']]

    # === ADD DATA === #

    tmp = None
    for txt in txt_col:
        if tmp is None:
            tmp = df[txt]
        else:
            tmp = tmp.str.cat(df[txt].fillna('') , sep='\n')

    df_data['text'] = tmp.fillna('')
    
    # remove people who died before current season
    idxRemove = np.zeros(df_data.shape[0],dtype=bool)
    for i, val in enumerate(df_data['Status'].values):
        char = df_data.index[i]
        if val is None or val is np.nan:
            continue
        if ('Dead' in val) | ('Deceased' in val):
            if s >= 5:
                continue
            elif len(re.findall('[' + str(s+1) + '-5]', df_data.loc[char, 'Season(s)'])) > 0:
                continue
            else:
                idxRemove[i] = True

    df_data = df_data.loc[~idxRemove]
    
    # now we have the data frame ... get X and y matrices
    X = df_data['text'].values

    # alive until proven dead !
    y = np.zeros(df_data.shape[0])
    for i, val in enumerate(df_data['Status'].values):
        if val is None or val is np.nan:
            continue
        # check if they have any seasons after the next one
        # this is to confirm that, if they died, they died in the next season
        if len(re.findall('[' + str(s+2) + '-' + str(np.max([s+2,5])) + ']',
                          df_data.iloc[i]['Season(s)'])) == 0:
            if 'Dead' in val:
                y[i] = 1.0
            elif 'Deceased' in val:
                y[i] = 1.0
        
    #print('Died in season {}'.format(s+1))
    #for i,val in enumerate(y):
    #    if val==1.0:
    #        print('{}'.format(df_data.index[i]))
    return X, y<jupyter_output><empty_output><jupyter_text>example for only one season```pythonX, y = extract_data_text(df, s=3)tf = fet.TfidfVectorizer(analyzer='word', ngram_range=(1,3), min_df = 0, stop_words = 'english')tfidf_matrix =  tf.fit_transform(X)feature_names = tf.get_feature_names()print('Example phrases:')for i in range(5):    print(feature_names[i+50])print('\n' + 'Shape of the design matrix: ' + '{}'.format(tfidf_matrix.shape))```<jupyter_code>seasons = [2,3,4,5]
scores = np.zeros([4,3]) # 4 seasons, 3fold CV

for s in range(4):
    X, y = extract_data_text(df, s=seasons[s])
    
    tf = fet.TfidfVectorizer(analyzer='word', ngram_range=(1,2), min_df = 0, stop_words = 'english')
    mdl = LassoCV(cv=5,fit_intercept=True)


    pipeline = Pipeline([
        ('tfidf', tf),
        ('clf', mdl),
    ])

    score = cross_val_score(pipeline, X, y, scoring='roc_auc')
    scores[s,:] = score
    
    print('Finished season {} of {}.'.format(seasons[s],seasons[-1]))
    print('Mean AUROC: {:0.3f}'.format(np.mean(score)))
seasons = [2,3,4,5]
scores = np.zeros([4,3]) # 4 seasons, 3fold CV

for s in [3]:
    X, y = extract_data_text(df, s=seasons[s])
    
    tf = fet.TfidfVectorizer(analyzer='word', ngram_range=(1,2), min_df = 0, stop_words = 'english')
    mdl = LassoCV(cv=5,fit_intercept=True)


    pipeline = Pipeline([
        ('tfidf', tf),
        ('clf', mdl),
    ])

    score = cross_val_score(pipeline, X, y, scoring='roc_auc')
    scores[s,:] = score
    
    print('Finished season {} of {}.'.format(seasons[s],seasons[-1]))
    print('Mean AUROC: {:0.3f}'.format(np.mean(score)))<jupyter_output>Finished season 5 of 5.
Mean AUROC: 0.500<jupyter_text>Last chance: the above code removed people who died before the current season. Let's try instead keeping all people, not including text from the season in which a character died.<jupyter_code>def extract_data_text_with_deaths(df, s=5):
    # given the dataframe and the season, extract some features    
    # by default, extract data up to and including season 3, i.e. s=3
    
    txt_col = ['Background','Season 1','Season 2','Season 3','Season 4','Season 5']
    
    txt_col = txt_col[0:(s+1)]
    
    # initialize the dataframe
    df_data = df.loc[:, ['Status','Season(s)']]

    # define the season in which each person died
    dead = np.zeros(df_data.shape[0],dtype=int)
    for i, val in enumerate(df_data['Status'].values):
        char = df_data.index[i]
        if val is None or val is np.nan:
            continue
        elif ('Dead' in val) | ('Deceased' in val):
            charDied = re.findall('[1-5]', df.loc[char, 'Season(s)'])
            dead[i] = float(charDied[-1])
                
    
    # === ADD DATA === #
    tmp = df[txt_col[0]].fillna('')
    
    for season, txt in enumerate(txt_col):
        # data is initialized with background, so skip this section
        if txt == 'Background':
            continue
        for i, char in enumerate(df_data.index):
            # if there is no text for the given season, skip
            if df.loc[char,txt] is None or df.loc[char,txt] is np.nan:
                continue
            # if the season is equal/after the season where the character died, don't add the text
            # note season is a numeric here
            elif season >= dead[i]:
                continue
            # otherwise, concatenate this season's text
            else:
                tmp[i] = tmp[i] + '\n' + (df.loc[char,txt])

    df_data['text'] = tmp
    df_data['died_in_season'] = dead

    # alive until proven dead !
    df_data['dead'] = (dead != 0) & (dead <= s)
    
    return df_data
seasons = [2,3,4,5]
scores = np.zeros([len(seasons),3]) # 4 seasons, 3fold CV
preds = np.zeros([df.shape[0],len(seasons)])

for i, s in enumerate(seasons):
    df_data = extract_data_text_with_deaths(df, s=s)
    
    # now we have the data frame ... get X and y matrices
    X = df_data['text'].values
    y = df_data['dead'].values

    score = cross_val_score(pipeline, X, y, scoring='roc_auc')
    pred = cross_val_predict(pipeline, X, y)
    scores[i,:] = score
    preds[:,i] = pred
    
    print('Finished season {} of {}.'.format(s,seasons[-1]))
    print('Mean AUROC: {:0.3f}'.format(np.mean(score)))<jupyter_output>Finished season 2 of 5.
Mean AUROC: 0.809
Finished season 3 of 5.
Mean AUROC: 0.776
Finished season 4 of 5.
Mean AUROC: 0.798
Finished season 5 of 5.
Mean AUROC: 0.832<jupyter_text>These AUROCs will probably be shifted upward ... but seems OK. Let's look at the top predictions it made for season 5.<jupyter_code>np.argsort?
i=3 # i=3 corresponds to season 5
sort_indices = np.argsort(-preds[:,i])

for n, idx in enumerate(sort_indices):
    print('{:30s} - {:0.2f} - {}'.format(df.index[idx], preds[idx,i], y[idx]))
    if n==5:
        break
# who didn't die but we thought should?

i=3 # i=3 corresponds to season 5
sort_indices = np.argsort(-preds[:,i])

for n, idx in enumerate(sort_indices):
    if y[idx] == False:
        print('{:20s} - {:2.2f}%'.format(df.index[idx], 100.0*preds[idx,i]))
        
    if np.sum(~y[sort_indices[0:n]])==10:
        break<jupyter_output>Robin_Arryn          - 94.07%
Tommen_Baratheon     - 74.46%
Rickon_Stark         - 72.01%
Tyrion_Lannister     - 67.81%
Daenerys_Targaryen   - 66.70%
Tysha                - 65.88%
Arya_Stark           - 63.93%
Bran_Stark           - 62.56%
Davos_Seaworth       - 54.56%
Sansa_Stark          - 53.89%
{'fraction_non_alphanumeric': 0.14294741276416487, 'fraction_numerical': 0.029909429193287932, 'mean_word_length': 2.735641227380016, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '555251', 'license': 'MIT', 'n_tokens_mistral': 5406, 'n_tokens_neox': 5019, 'n_words': 1293, 'path': 'got-mortality-prediction.ipynb', 'repo_name': 'alistairewj/got-predictions', 'size': '14246'}