<commit_before>import sys
import json

from ott.utils.config_util import ConfigUtil
from ott.utils.cache_base import CacheBase
from ott.loader.gtfs.cache import Cache

class Load(CacheBase):
    """ load GTFS data into a gtfsdb
    """
    gtfs_zip_files  = None

    def __init__(self, config=None, force_reload=False):
        super(CacheBase, self).__init__()
        reload = force_reload
        self.config.section = 'gtfs'

        feeds = config.get_json('feeds')
        if Cache.check_gtfs_files_against_cache(feeds, self.cache_dir):
            reload = True
        if reload:
            # TODO
            print "TODO ... implement load of gtfsdb"

def main(argv=sys.argv):
    import pdb; pdb.set_trace()
    Load()

if __name__ == '__main__':
    main()
<commit_msg>config reader part of gtfsdb loader now<commit_after>import sys

from ott.utils.cache_base import CacheBase
from ott.loader.gtfs.cache import Cache

class Load(CacheBase):
    """ load GTFS data into a gtfsdb
    """
    gtfs_zip_files  = None

    def __init__(self, config=None, force_reload=False):
        super(Load, self).__init__(section='gtfs')
        reload = force_reload

        feeds = self.config.get_json('feeds')
        if Cache.check_gtfs_files_against_cache(feeds, self.cache_dir):
            reload = True
        if reload:
            # TODO
            print "TODO ... implement load of gtfsdb"

def main(argv=sys.argv):
    #import pdb; pdb.set_trace()
    Load()

if __name__ == '__main__':
    main()

{'fraction_non_alphanumeric': 0.10059563203176704, 'fraction_numerical': 0.0, 'mean_word_length': 2.7333333333333334, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6580525', 'n_tokens_mistral': 499, 'n_tokens_neox': 489, 'n_words': 122}