<jupyter_start><jupyter_text>Actual Transformation<jupyter_code># assert False
import os
input_files = os.listdir('dataset/individual_match')
count = 0
for fileName in input_files:
    try:
        transformFile(fileName)
        f_reload = pd.read_csv('dataset/individual_match_m1/mod_batsman_stats_'+fileName, index_col=None, header=0)
        f_reload = pd.read_csv('dataset/individual_match_m1/mod_bowler_stats_'+fileName, index_col=None, header=0)
    except RuntimeError as e:
        print(e)
        count += 1
print("Total files failed ", count)<jupyter_output>Total files failed  0<jupyter_text>Merge all into One<jupyter_code># Merging all batsman
import os
mod_files = os.listdir('dataset/individual_match_m1')
li = []
for filename in mod_files:
    if('batsman' not in filename):
        continue
    df = pd.read_csv('dataset/individual_match_m1/'+filename, index_col=None, header=0)
    li.append(df)

frame = pd.concat(li, axis=0, ignore_index=True)
print(frame.shape)
frame.to_csv('dataset/all_matches_m1_batsman.csv', index=False, na_rep="NaN")
# Merging all batsman
# assert False
import os
mod_files = os.listdir('dataset/individual_match_m1')
li = []
for filename in mod_files:
    if('bowler' not in filename):
        continue
    df = pd.read_csv('dataset/individual_match_m1/'+filename, index_col=None, header=0)
    li.append(df)

frame = pd.concat(li, axis=0, ignore_index=True)
print(frame.shape)
frame.to_csv('dataset/all_matches_m1_bowler.csv', index=False, na_rep="NaN")<jupyter_output><empty_output><jupyter_text>Final Data Population<jupyter_code>df = pd.read_csv('dataset/all_matches.csv')
print(df.shape)
df.columns
striker_bowler = df.groupby(['venue', 'innings', 'match_id','season', 'striker', 'bowler']).agg(
    balls = ('ball', 'count'),
    runs_bat = ('runs_off_bat', 'sum'),
    extras = ('extras', 'sum'),
).reset_index().sort_values(by='match_id')
striker_bowler[:1]
bowler_stats = pd.read_csv('dataset/all_matches_m1_bowler.csv')
batsman_stats = pd.read_csv('dataset/all_matches_m1_batsman.csv')
b = batsman_stats.query('striker == "{0}" and season == "{1}" and match_id < {2}'
                        .format("<NAME>",'2009', 392218)).drop(
    ['striker','match_id','venue','season','innings', 'balls_played'], axis=1)
b
for row in striker_bowler[:1].iterrows():
    rf = row[1]
    print(rf)
    
    bt = batsman_stats.query('striker == "{0}" and season == "{1}" and match_id < {2}'
                             .format(rf.striker, rf.season, rf.match_id)).mean()
    bo = bowler_stats.query('bowler == "{0}" and season == "{1}" and match_id < {2}'
                             .format(rf.bowler, rf.season, rf.match_id)).mean()
    
    rf = rf.append(b[3:-1])
    print(rf)
b.vstack(b)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15390173410404626, 'fraction_numerical': 0.015895953757225433, 'mean_word_length': 4.483168316831684, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '471163', 'license': 'CC0-1.0', 'n_tokens_mistral': 1030, 'n_tokens_neox': 1000, 'n_words': 179, 'path': 'iit-madras/Cricket Hackathon 2021/M1_batch generation.ipynb', 'repo_name': 'rohitnandi12/ai-hackathons', 'size': '2769'}