<jupyter_start><jupyter_text>Constructing Datasets with Only hasball DataIn this notebook, we create csv files which contains hasball data for each second of the match.<jupyter_code>import pandas as pd
from os import listdir
from os.path import join
import re
raw_dir = '../data/processed'
out_dir = '../data/hasball'

out_cols = ['half', 'minute', 'second', 'teamPoss', 'hasballTeam']

raw_regex = re.compile(r'\d+_raw.csv')
from utils import remove_missing_raw_rows


def raw_to_hasball(raw_filename):
    """
    Creates a hasball csv file in out_dir from a given raw csv filename.
    
    Parameters
    ----------
    raw_filename: Name of raw csv file in raw_dir.
    """
    df = pd.read_csv(join(raw_dir, raw_filename))
    df = remove_missing_raw_rows(df)
    data = df[out_cols].values
    data_uniq = np.unique(data, axis=0)
    out_df = pd.DataFrame(data=data_uniq, columns=out_cols)
    
    match_id = raw_filename.split('_')[0]
    out_file = '{}_hasball.csv'.format(match_id)
    out_df.to_csv(join(out_dir, out_file), index=False)
import multiprocessing


raw_filenames = (f for f in listdir(raw_dir) if raw_regex.match(f))

pool = multiprocessing.Pool()
pool.map(raw_to_hasball, raw_filenames);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11584875301689461, 'fraction_numerical': 0.0016090104585679806, 'mean_word_length': 4.316239316239316, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '143746', 'license': 'Apache-2.0', 'n_tokens_mistral': 443, 'n_tokens_neox': 436, 'n_words': 109, 'path': 'notebooks/hasball_from_raw.ipynb', 'repo_name': 'eozd/SIU-2018', 'size': '1243'}