<jupyter_start><jupyter_text>There are 171 people who double and actors and directors<jupyter_code>sum(el in directors for el in actors)
%%time
total_gross = []
mean_score = []
mean_votes = []
for i in df.index:
    if df.is_director[i]:
        temp = movie_industry[movie_industry.director == df.agent[i]]
        temp = temp[temp.year <= df.year[i]]
        if len(temp) > 0 :  
            gross = sum(temp.gross.values)
            score = np.mean(temp.score.values)
            votes = np.mean(temp.votes.values)
        else:
            gross = 0
            score = 0
            votes = 0
        total_gross.append(gross)
        mean_score.append(score)
        mean_votes.append(votes)
    else:
        temp = movie_industry[movie_industry.star == df.agent[i]]
        temp = temp[temp.year <= df.year[i]]
        if len(temp) > 0 :  
            gross = sum(temp.gross.values)
            score = np.mean(temp.score.values)
            votes = np.mean(temp.votes.values)
        else:
            gross = 0
            score = 0
            votes = 0
        total_gross.append(gross)
        mean_score.append(score)
        mean_votes.append(votes)
df["log_total_gross"] = [np.log(i+1) for i in total_gross]
df["mean_score"] = mean_score
df["mean_votes"] = mean_votes
df
temp = movie_industry.groupby("genre").count().sort_values("director", ascending= False)
temp
sum(temp.budget.values[0:8])/ sum(temp.budget.values)
genre = temp.index.values[0:8]
genre
%%time
d = {}
for gen in genre:
    d[gen] = []   
for i in df.index:
    if df.is_director[i]:
        temp = movie_industry[movie_industry.director == df.agent[i]]
    else:
        temp = movie_industry[movie_industry.star == df.agent[i]]
    temp = temp[temp.year <= df.year[i]]
    if len(temp) > 0:
        temp = temp.groupby(["genre"])["director"].count()
        for gen in genre:
            if gen in temp.index:
                d[gen].append(temp[gen])
            else:
                d[gen].append(0)
    else:
        for gen in genre:
            d[gen].append(0)
temp = pd.DataFrame(d)
df = df.merge(temp, left_index =True, right_index =True)
df
temp = movie_industry.groupby("rating").count().sort_values("director", ascending= False)
temp
sum(temp.budget.values[0:5])/ sum(temp.budget.values)
ratings = temp.index.values[0:5]
ratings
%%time
d = {}
for gen in ratings:
    d[gen] = []   
for i in df.index:
    if df.is_director[i]:
        temp = movie_industry[movie_industry.director == df.agent[i]]
    else:
        temp = movie_industry[movie_industry.star == df.agent[i]]
    temp = temp[temp.year <= df.year[i]]
    if len(temp) > 0:
        temp = temp.groupby(["rating"])["director"].count()
        for gen in ratings:
            if gen in temp.index:
                d[gen].append(temp[gen])
            else:
                d[gen].append(0)
    else:
        for gen in ratings:
            d[gen].append(0)
temp = pd.DataFrame(d)
df = df.merge(temp, left_index =True, right_index =True)
df
oscar = pd.read_csv("../data/the_oscar_award.csv")
oscar.category = oscar.category.str.lower()
filtered = oscar[oscar.category.str.contains("(actor)|(actress)|(directing)", regex = True)]
filtered.loc[filtered.category.str.contains("(actor)|(actress)"), 'is_director'] = 0
filtered.loc[filtered.category.str.contains("directing"), 'is_director'] = 1
oscar_ppl = filtered[~filtered.name.str.contains("( and)|(,)|(and )", regex = True)]
oscar_ppl.is_director = oscar_ppl.is_director.astype(np.uint8)
oscar_ppl.head()
oscar_ppl.shape
filtered[filtered.name.str.contains("( and)|(,)|(and )", regex = True)]
#Manually modify the rows where there are special characters (for movie after 1986)
oscar_ppl = pd.concat([oscar_ppl, filtered[(filtered.index==7558) | (filtered.index==6506)]])
oscar_ppl.shape
oscar_ppl[oscar_ppl.name=="<NAME> and <NAME>"]
oscar_ppl = pd.concat([oscar_ppl, filtered[(filtered.index==8832) | (filtered.index==9184)]], ignore_index=False)
oscar_ppl = pd.concat([oscar_ppl, filtered[(filtered.index==8832) | (filtered.index==9184)]], ignore_index=True)
oscar_ppl.shape
oscar_ppl[oscar_ppl.name== "<NAME> and <NAME>"]
oscar_ppl.at[2194, 'name'] = '<NAME>'
oscar_ppl.at[2195, 'name'] = '<NAME>'
oscar_ppl.at[2196, 'name'] = '<NAME>'
oscar_ppl.at[2197, 'name'] = '<NAME>'
oscar_ppl.shape
oscar_ppl[oscar_ppl.name=='<NAME>'].index
oscar_ppl.at[2062, 'name'] = '<NAME>'
oscar_ppl = oscar_ppl[oscar_ppl['year_film']>1985]
print('oscar award dataset has shape {}'.format(oscar_ppl.shape))
oscar_ppl.head()
%%time
total_nominations =[]
total_win_counts=[]
for i in df.index:
    if df.is_director[i]:
        temp = oscar_ppl[(oscar_ppl.name == df.agent[i]) & (oscar_ppl.is_director == True)]
        temp = temp[temp.year_film <= df.year[i]]
        nomination = temp.shape[0]
        total_nominations.append(nomination)
        win = sum(temp.winner.values)
        total_win_counts.append(win)
    else:
        temp = oscar_ppl[(oscar_ppl.name == df.agent[i]) & (oscar_ppl.is_director == False)]
        temp = temp[temp.year_film <= df.year[i]]
        nomination = temp.shape[0]
        total_nominations.append(nomination)
        win = sum(temp.winner.values)
        total_win_counts.append(win)
df["nominations"] = total_nominations
df["win_counts"] = total_win_counts
df.head()
df
df.to_csv("../data/features.csv", index= False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1571216065451841, 'fraction_numerical': 0.015433246560059502, 'mean_word_length': 3.225451688923802, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '135575', 'license': 'MIT', 'n_tokens_mistral': 2031, 'n_tokens_neox': 1954, 'n_words': 352, 'path': 'notebooks/Recent10_covariate_gen.ipynb', 'repo_name': 'marlinfiggins/West-Coast-Datathon-F2020', 'size': '5442'}