<jupyter_start><jupyter_text><jupyter_code>import glob
import os

import pandas as pd
from yaml import safe_load

if 'google.colab' in str(get_ipython()):
  from google.colab import drive
  drive.mount('/content/drive')
  
  BASE_PATH = 'drive/MyDrive/git_data/raw_data'
else:
  BASE_PATH = "./"

FILE_LIST = [os.path.join(BASE_PATH, f) for f in os.listdir(BASE_PATH)]

print(FILE_LIST[0])

match_info_df = pd.DataFrame()

for i, fname in enumerate(FILE_LIST):
  with open(fname, 'r') as f:
    data = safe_load(f)#["innings"][0]["1st innings"]["deliveries"]
    meta_data = pd.json_normalize(data)
    
    meta_data["match_id"] = i

    match_info_df = match_info_df.append(meta_data)
    # innings = pd.json_normalize(safe_load(f))

  if (i + 1) % 25 == 0:
    print("Proceesed {}".format(i))

columns = [
           "city",
           "dates",
           "outcome_by_runs",
           "method",
           "winner",
           "overs",
           "player_of_match",
           "teams",
           "toss",
           "toss_winner",
           "umpires",
           "venue",
           "match_id",
           "outcome_by_wickets"
]

match_info_df = match_info_df.drop(columns = match_info_df.iloc[:, [0,1,2,3,5,7,8]].columns)
match_info_df.columns = columns
match_info_df = match_info_df.set_index("match_id")
match_info_df.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14317180616740088, 'fraction_numerical': 0.010279001468428781, 'mean_word_length': 3.0088235294117647, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '456796', 'license': 'MIT', 'n_tokens_mistral': 529, 'n_tokens_neox': 516, 'n_words': 87, 'path': 'Data_Processing.ipynb', 'repo_name': 'rikithreddy/ipl-analysis', 'size': '1362'}