<commit_before>'''
Parse matches in json format to a cvs file.

The matches are looked up by participants. Only 22 statistical attributes of
participants selected by ONG et al. (2015) are considered. No transformation is
performed on the data as normalization/standardization.
'''
import json
import os
from datetime import datetime
from config import DUMP_DIR, DATA_DIR


def parse(value):
    '''
    Parse a value of any python type to string.

    Boolean values are converted to integer values {True: 1, False: 0}.
    '''
    return parse(int(value)) if isinstance(value, bool) else str(value)


def csvrow(*values):
    '''
    Create a csv row.

    Convert a list of values (any python type) into a comma separated values
    string.
    '''
    return ','.join(map(parse, values))


def load_match(filename):
    '''
    Load a specific match in json format.
    '''
    with open(filename) as f:  # open match
        return json.load(f)  # read match

filename = 'data.%s.csv' % datetime.now().strftime('%Y%m%d.%H%M%S')
csvfile = open(DATA_DIR + filename, 'w+')

# general info - not used for clustering
info_attrs = [
    'matchId',
    'matchCreation',
    'summonerId',
    'championId',
]

# the 22 statistical attributes of participants selected by Ong et al. (2015)
stats_attrs = [
    # booleans attributes
    'winner',
    'firstBloodKill',
    'firstTowerKill',
    'firstTowerAssist',
    # numeric attributes:
    'kills',
    'assists',
    'deaths',
    'goldEarned',
    'totalDamageDealt',
    'magicDamageDealt',
    'physicalDamageDealt',
    'totalDamageDealtToChampions',
    'totalDamageTaken',
    'minionsKilled',
    'neutralMinionsKilled',
    'totalTimeCrowdControlDealt',
    'wardsPlaced',
    'towerKills',
    'largestMultiKill',
    'largestKillingSpree',
    'largestCriticalStrike',
    'totalHeal'
]

headers = ','.join('\"%s\"' % header for header in info_attrs + stats_attrs)

csvfile.write(headers)
csvfile.write('\n')

for f in os.listdir(DUMP_DIR):  # list matches

    data = load_match(DUMP_DIR+f)

    # looking up by participants
    for i, participant in enumerate(data['participants']):

        # general info values
        info = [
            data['matchId'],
            data['matchCreation'],
            data['participantIdentities'][i]['player']['summonerId'],
            participant['championId']
        ]

        # statistical values. Filtering from all participant stats
        stats = [participant['stats'][attr] for attr in stats_attrs]

        csvfile.write(csvrow(*(info + stats)))  # write values in csv format
        csvfile.write('\n')
<commit_msg>Minor changes in parser.<commit_after>'''
Parse matches in json format to a cvs file.

The matches are looked up by participants. Only 22 statistical attributes of
participants selected by ONG et al. (2015) are considered. No transformation is
performed on the data as normalization/standardization.
'''
import json
import os
from datetime import datetime
from config import DUMP_DIR, DATA_DIR


def parse(value):
    '''
    Parse a value of any python type to string.

    Boolean values are converted to integer values {True: 1, False: 0}.
    '''
    return parse(int(value)) if isinstance(value, bool) else str(value)


def csvrow(*values):
    '''
    Create a csv row.

    Convert a list of values (any python type) into a comma separated values
    string.
    '''
    return ','.join(map(parse, values))


def load_match(filename):
    '''
    Load a specific match in json format.
    '''
    with open(filename) as f:  # open match
        return json.load(f)  # read match

filename = 'data.%s.csv' % datetime.now().strftime('%Y%m%d.%H%M%S')
csvfile = open(DATA_DIR + filename, 'w+')
print('csv file:', csvfile.name)

# general info - not used for clustering
info_attrs = [
    'matchId',
    'matchCreation',
    'summonerId',
    'championId',
]

# the 22 statistical attributes of participants selected by Ong et al. (2015)
stats_attrs = [
    # booleans attributes
    'winner',
    'firstBloodKill',
    'firstTowerKill',
    'firstTowerAssist',
    # numeric attributes:
    'kills',
    'assists',
    'deaths',
    'goldEarned',
    'totalDamageDealt',
    'magicDamageDealt',
    'physicalDamageDealt',
    'totalDamageDealtToChampions',
    'totalDamageTaken',
    'minionsKilled',
    'neutralMinionsKilled',
    'totalTimeCrowdControlDealt',
    'wardsPlaced',
    'towerKills',
    'largestMultiKill',
    'largestKillingSpree',
    'largestCriticalStrike',
    'totalHeal'
]

headers = ','.join('\"%s\"' % header for header in info_attrs + stats_attrs)

csvfile.write(headers)
csvfile.write('\n')

for f in os.listdir(DUMP_DIR):  # list matches

    if not f.endswith('.json'):
        continue

    data = load_match(DUMP_DIR+f)

    # looking up by participants
    for i, participant in enumerate(data['participants']):

        # general info values
        info = [
            data['matchId'],
            data['matchCreation'],
            data['participantIdentities'][i]['player']['summonerId'],
            participant['championId']
        ]

        # statistical values. Filtering from all participant stats
        stats = [participant['stats'][attr] for attr in stats_attrs]

        csvfile.write(csvrow(*(info + stats)))  # write values in csv format
        csvfile.write('\n')

{'fraction_non_alphanumeric': 0.11827354260089686, 'fraction_numerical': 0.00523168908819133, 'mean_word_length': 3.18203125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4008995', 'n_tokens_mistral': 1728, 'n_tokens_neox': 1628, 'n_words': 484}