<commit_before>import os
import logging
import logging.config
log = logging.getLogger(__file__)


from ott.utils import file_utils
from ott.utils import object_utils
from ott.utils import web_utils
from ott.utils.cache_base import CacheBase

from ott.loader.gtfs.info import Info
from ott.loader.gtfs.diff import Diff


class GtfsCache(CacheBase):
    """ Does a 'smart' cache of a gtfs file
         1. it will look to see if a gtfs.zip file is in the cache, and download it and put it in the cache if not
         2. once cached, it will check to see that the file in the cache is the most up to date data...
    """
    feeds = []

    def __init__(self):
        super(GtfsCache, self).__init__(section='gtfs')
        self.feeds = self.config.get_json('feeds')

    def check_cached_feeds(self, force_update=False):
        for f in self.feeds:
            url,name = GtfsCache.get_url_filename(f)
            self.check_feed(url, name, force_update)

    def check_feed(self, url, file_name, force_update=False):
        ''' download feed from url, and check it against the cache
            if newer, then replace cached feed .zip file with new version
        '''
        # step 1: file name
        file_name = file_name
        file_path = os.path.join(self.cache_dir, file_name)

        # step 2: download new gtfs file
        url = url
        tmp_path = os.path.join(self.tmp_dir, file_name)

        # step 2b: don't keep downloading a file ... make sure the tmp file is at least 3 hours
        if os.path.exists(tmp_path) is False or file_utils.file_age_seconds(tmp_path) > 10000:
            web_utils.wget(url, tmp_path)

        # step 3: check the cache whether we should update or not
        update = force_update
        if not force_update:
            if self.is_fresh_in_cache(file_path):
                log.info("diff {} against cached {}".format(tmp_path, file_path))
                diff = Diff(file_path, tmp_path)
                if diff.is_different():
                    update = True
            else:
                update = True

        # step 4: mv old file to backup then mv new file in tmp dir to cache
        if update:
            log.info("move {} to cache {}".format(tmp_path, file_path))
            file_utils.bkup(file_path)
            file_utils.cp(tmp_path, file_path)

    def cmp_file_to_cached(self, gtfs_zip_name, cmp_dir):<commit_msg>fixes download logic of gtfs<commit_after>import os
import logging
import logging.config
log = logging.getLogger(__file__)


from ott.utils import file_utils
from ott.utils import object_utils
from ott.utils import web_utils
from ott.utils.cache_base import CacheBase

from ott.loader.gtfs.info import Info
from ott.loader.gtfs.diff import Diff


class GtfsCache(CacheBase):
    """ Does a 'smart' cache of a gtfs file
         1. it will look to see if a gtfs.zip file is in the cache, and download it and put it in the cache if not
         2. once cached, it will check to see that the file in the cache is the most up to date data...
    """
    feeds = []

    def __init__(self):
        super(GtfsCache, self).__init__(section='gtfs')
        self.feeds = self.config.get_json('feeds')

    def check_cached_feeds(self, force_update=False):
        for f in self.feeds:
            url,name = GtfsCache.get_url_filename(f)
            self.check_feed(url, name, force_update)

    def check_feed(self, url, file_name, force_update=False):
        ''' download feed from url, and check it against the cache
            if newer, then replace cached feed .zip file with new version
        '''
        # step 1: file name
        file_name = file_name
        file_path = os.path.join(self.cache_dir, file_name)

        # step 2: download new gtfs file
        url = url
        tmp_path = os.path.join(self.tmp_dir, file_name)

        # step 2b: don't keep downloading a file ... make sure the tmp file is at least 2 hours
        if os.path.exists(tmp_path) is False or \
           file_utils.file_age_seconds(tmp_path) > 7200:
            web_utils.wget(url, tmp_path)

        # step 3: check the cache whether we should update or not
        update = force_update
        if not force_update:
            if self.is_fresh_in_cache(file_path):
                log.info("diff {} against cached {}".format(tmp_path, file_path))
                diff = Diff(file_path, tmp_path)
                if diff.is_different():
                    update = True
            else:
                update = True

        # step 4: mv old file to backup then mv new file in tmp dir to cache
        if update:
            log.info("cp {} to cache {}".format(tmp_path, file_path))
            file_utils.bkup(file_path)
            file_utils.cp(tmp_path, file_path)

    def cmp_file_to_cached(self, gtfs_zip_name, cmp_dir):
{'fraction_non_alphanumeric': 0.08037383177570094, 'fraction_numerical': 0.005192107995846314, 'mean_word_length': 2.349095966620306, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7123772', 'n_tokens_mistral': 1490, 'n_tokens_neox': 1441, 'n_words': 487}