<commit_before>import os, re
from collections import Counter


def get_card_data():
    military_targets = (
        ' NOVELTY',
        ' RARE',
        ' GENE',
        ' ALIEN',
        ' AGAINST_REBEL',
        ' XENO',
        )
    card_data = {}
    with open('cards.txt', 'r') as card_file:
        for line in card_file:
            if line.startswith('N:'):
                card_name = line[2:].strip()
                card_data[card_name] = {'military': Counter()}
            elif line.startswith('T:'):
                card_data[card_name]['raw_VP'] = int(line.split(':')[-1])
            elif line.startswith('G:'):
                card_data[card_name]['goods'] = line.split(':')[1].lower().strip()
            elif line.startswith('P:3') and 'EXTRA_MILITARY' in line:
                # Cut the phase identifier and the last number I don't understand.
                line = line[4:-3]
                bonus = int(line.split(':')[1])
                potential = False
                if 'CONSUME' in line or 'DISCARD' in line:
                    potential = True
                specialized = False
                for target in military_targets:
                    if target in line:
                        target = target.strip().lower()
                        break
                    target = 'normal'
                target = target if target != 'against_rebel' else 'rebel'
                key = ('' if not potential else 'potential_') + target
                card_data[card_name]['military'][key] = bonus
    return card_data


logs = [f for f in os.listdir('.') if f.startswith('export_')]

log = max(logs)
print('Processing {} ...'.format(log))

<commit_msg>more complete card data parsing<commit_after>import os, re
from collections import Counter


def get_card_data():
    military_targets = (
        ' NOVELTY',
        ' RARE',
        ' GENE',
        ' ALIEN',
        ' AGAINST_REBEL',
        ' XENO',
        )
    card_data = {}
    with open('cards.txt', 'r') as card_file:
        card = None
        for line in card_file:
            if line.startswith('N:'):
                card_name = line[2:].strip()
                card_data[card_name] = {
                        'military': Counter(),
                        'flags': set(),
                        '?_VP': [],
                        }
                card = card_data[card_name]
            elif line.startswith('T:'):
                card_type, cost, vp = line.split(':')[1:]
                cost, vp = int(cost), int(vp)
                card['raw_VP'] = vp
                card['cost'] = cost
                card['flags'].add('WORLD' if card_type == '1' else 'DEVEL')
            elif line.startswith('G:'):
                goods = line.split(':')[1].strip()
                card['goods'] = goods.lower()
            elif line.startswith('F:'):
                flags = line[2:].split('|')
                card['flags'] |= {flag.strip() for flag in flags}
                if card['cost'] == 6:
                    card['flags'].add('SIX')
                if card['flags'] & {'WORLD', 'MILITARY'} == {'WORLD'}:
                    card['flags'].add('NONMILITARY')
                if card.get('goods') and 'WINDFALL' not in card['flags']:
                    card['flags'].add('PRODUCTION')
            elif line.startswith('P:3') and 'EXTRA_MILITARY' in line:
                line = line[4:-3]
                bonus = int(line.split(':')[1])
                potential = False
                if 'CONSUME' in line or 'DISCARD' in line:
                    potential = True
                for target in military_targets:
                    if target in line:
                        target = target.strip().lower()
                        break
                    target = 'normal'
                target = target if target != 'against_rebel' else 'rebel'
                key = ('' if not potential else 'potential_') + target
                card['military'][key] = bonus
            elif line.startswith('V:'):
                vp, code, name = line.split(':')[1:]
                code = code.replace('_FLAG', '')
                if code in ('ALIEN_TECHNOLOGY', 'ALIEN_SCIENCE', 'ALIEN_UPLIFT'):
                    continue;
                elif code == 'NAME':
                    code = {name.strip()}
                elif code in ('THREE_VP', 'NEGATIVE_MILITARY', 'TOTAL_MILITARY'):
                    code = {code}
                elif code.startswith('ANTI_XENO'):
                    a = 'ANTI_XENO'
                    code = {w.lstrip('_') for w in code.partition(a) if w}
                else:
                    code = set(code.split('_'))
                card['?_VP'].append((code, vp))
    return card_data


logs = [f for f in os.listdir('.') if f.startswith('export_')]

log = max(logs)
print('Processing {} ...'.format(log))


{'fraction_non_alphanumeric': 0.13870568837592745, 'fraction_numerical': 0.003709810387469085, 'mean_word_length': 1.4423754403623552, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2913911', 'n_tokens_mistral': 1457, 'n_tokens_neox': 1366, 'n_words': 327}