<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os

from parsers.main_parser import MainParser


if __name__ == '__main__':

    src = r"D:\nhl\official_and_json\2016-17\2017-02\2017-02-01.zip"
    # src = r"D:\nhl\official_and_json\2016-17\2017-02\2017-02-20"
    # src = r"D:\nhl\official_and_json\_2015-16\2016-05\2016-05-09.zip"

    src_dir = r"D:\nhl\official_and_json\_2014-15\2014-12"
    src_dir = r"D:\nhl\official_and_json\2016-17\2016-10"
    src_dir = r"D:\nhl\official_and_json\_2015-16\2016-04"

    files = os.listdir(src_dir)
    files = [src]

    # for f in files[17:18]:
    for f in files[:1]:
        print(f)
        if not os.path.splitext(f)[-1].lower().endswith(".zip"):
            continue
        src = os.path.join(src_dir, f)
        mp = MainParser(src)
        print(mp.game_ids)

        for game_id in mp.game_ids[:1]:
            mp.parse_single_game(game_id)

<commit_msg>Utilize disposal function for parser object<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os

from parsers.main_parser import MainParser


if __name__ == '__main__':

    src = r"D:\nhl\official_and_json\2016-17\2017-02\2017-02-01.zip"
    # src = r"D:\nhl\official_and_json\2016-17\2017-02\2017-02-20"
    # src = r"D:\nhl\official_and_json\_2015-16\2016-05\2016-05-09.zip"
    src = r"D:\nhl\official_and_json\_2015-16\2016-04\2016-04-10.zip"

    src_dir = r"D:\nhl\official_and_json\_2014-15\2014-12"
    src_dir = r"D:\nhl\official_and_json\2016-17\2016-10"
    src_dir = r"D:\nhl\official_and_json\_2015-16\2016-04"

    files = os.listdir(src_dir)
    files = [src]

    # for f in files[17:18]:
    for f in files[:1]:
        print(f)
        if not os.path.splitext(f)[-1].lower().endswith(".zip"):
            continue
        src = os.path.join(src_dir, f)
        mp = MainParser(src)
        print(mp.game_ids)

        for game_id in mp.game_ids[:]:
            mp.parse_single_game(game_id)

        mp.dispose()

{'fraction_non_alphanumeric': 0.16111111111111112, 'fraction_numerical': 0.11464646464646465, 'mean_word_length': 2.5822784810126582, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '601944', 'n_tokens_mistral': 967, 'n_tokens_neox': 792, 'n_words': 114}