<commit_before>""" Run build.py weekly (or daily) on the test maps servers (e.g., maps6 / maps10).  I will
    download the latest gtfs.zip file, check to see if it's newer than the last version
    I downloaded, and if so, build a new OTP Graph.

    @see deploy.py, which is a companion script that runs on the production servers, and
    deploys a new OTP graph into production.
"""
import os
import sys
import time
import logging
log = logging.getLogger(__file__)

from ott.utils import otp_utils
from ott.utils import file_utils
from ott.utils.cache_base import CacheBase

from ott.loader.gtfs.gtfs_cache import GtfsCache
from ott.loader.osm.osm_cache import OsmCache
from ott.loader.gtfs.info import Info
from ott.loader.otp.preflight.test_runner import TestRunner


class OtpBuilder(CacheBase):
    """ build an OTP graph
    """
    feeds = None
    graphs = None
    expire_days = 45

    graph_name = otp_utils.GRAPH_NAME
    graph_size = 35000000
    graph_failed = graph_name + "-failed-tests"

    def __init__(self, force_update=False, dont_update=False):
        super(OtpBuilder, self).__init__('otp')
        self.feeds = self.config.get_json('feeds', section='gtfs')
        self.graphs = self.config_graph_dirs(force_update, dont_update)

    def config_graph_dirs(self, force_update=False, dont_update=False):
        """ read the config for graph specs like graph dir and web port (for running OTP)
            this routine will gather config .json files, .osm files and gtfs .zips into the graph folder
        """
        graphs = otp_utils.get_graphs(self)

        # check for config list of graphs ... create a default if nothing exists
        if graphs is None or len(graphs) == 0:
            graphs = [otp_utils.get_graph_details(None)]  # returns a default graph config

        # run thru the graphs and
        for g in graphs:
            graph_dir = otp_utils.config_graph_dir(g, self.this_module_dir)
            filter = g.get('filter')
            if force_update or not dont_update:
                OsmCache.check_osm_file_against_cache(graph_dir)
                GtfsCache.check_feeds_against_cache(self.feeds, graph_dir, force_update, filter)
        return graphs

    def build_and_test_graphs(self, java_mem=None, force_update=False, start_server=True):
        """ will build and test each of the graphs we have in self.graphs
        """
        ret_val = True
        for g in self.graphs:
            success, rebuilt = self.build_graph(g['dir'], java_mem, force_update)
            if success and rebuilt:
                success = self.test_graph(graph=g, java_mem=java_mem, start_server=start_server)
                ret_val = success
            elif not success:
                ret_val = False
                log.warn("graph build failed for graph {}".format(g['name']))
        return ret_val

    def only_test_graphs(self, java_mem=None, break_on_fail=False, start_server=True):
        """ will test each of the graphs we have in self.graphs
        """
        ret_val = True
        for g in self.graphs:
            success = self.test_graph(graph=g, java_mem=java_mem, start_server=start_server)
            if not success:
                ret_val = False
                if break_on_fail:
                    break
        return ret_val

    def build_graph(self, graph_dir, java_mem=None, force_update=False):
        """ will rebuild the graph...
        """
        success = True

        # step 1: set some params
        rebuild_graph = force_update

        # step 2: check graph file is fairly recent and properly sized
        graph_path = os.path.join(graph_dir, self.graph_name)
        if not file_utils.exists_and_sized(graph_path, self.graph_size):
            rebuild_graph = True

        # step 3: check the cache files
        if file_utils.dir_has_newer_files(graph_path, graph_dir, include_filter=".jar,.json,.osm,.zip"):
            rebuild_graph = True<commit_msg>fix false timeouts for different files<commit_after>""" Run build.py weekly (or daily) on the test maps servers (e.g., maps6 / maps10).  I will
    download the latest gtfs.zip file, check to see if it's newer than the last version
    I downloaded, and if so, build a new OTP Graph.

    @see deploy.py, which is a companion script that runs on the production servers, and
    deploys a new OTP graph into production.
"""
import os
import sys
import time
import logging
log = logging.getLogger(__file__)

from ott.utils import otp_utils
from ott.utils import file_utils
from ott.utils.cache_base import CacheBase

from ott.loader.gtfs.gtfs_cache import GtfsCache
from ott.loader.osm.osm_cache import OsmCache
from ott.loader.gtfs.info import Info
from ott.loader.otp.preflight.test_runner import TestRunner


class OtpBuilder(CacheBase):
    """ build an OTP graph
    """
    feeds = None
    graphs = None
    expire_days = 45

    graph_name = otp_utils.GRAPH_NAME
    graph_size = 35000000
    graph_failed = graph_name + "-failed-tests"

    def __init__(self, force_update=False, dont_update=False):
        super(OtpBuilder, self).__init__('otp')
        self.feeds = self.config.get_json('feeds', section='gtfs')
        self.graphs = self.config_graph_dirs(force_update, dont_update)

    def config_graph_dirs(self, force_update=False, dont_update=False):
        """ read the config for graph specs like graph dir and web port (for running OTP)
            this routine will gather config .json files, .osm files and gtfs .zips into the graph folder
        """
        graphs = otp_utils.get_graphs(self)

        # check for config list of graphs ... create a default if nothing exists
        if graphs is None or len(graphs) == 0:
            graphs = [otp_utils.get_graph_details(None)]  # returns a default graph config

        # run thru the graphs and
        for g in graphs:
            graph_dir = otp_utils.config_graph_dir(g, self.this_module_dir)
            filter = g.get('filter')
            if force_update or not dont_update:
                OsmCache.check_osm_file_against_cache(graph_dir)
                GtfsCache.check_feeds_against_cache(self.feeds, graph_dir, force_update, filter)
        return graphs

    def build_and_test_graphs(self, java_mem=None, force_update=False, start_server=True):
        """ will build and test each of the graphs we have in self.graphs
        """
        ret_val = True
        for g in self.graphs:
            success, rebuilt = self.build_graph(g['dir'], java_mem, force_update)
            if success and rebuilt:
                success = self.test_graph(graph=g, java_mem=java_mem, start_server=start_server)
                ret_val = success
            elif not success:
                ret_val = False
                log.warn("graph build failed for graph {}".format(g['name']))
        return ret_val

    def only_test_graphs(self, java_mem=None, break_on_fail=False, start_server=True):
        """ will test each of the graphs we have in self.graphs
        """
        ret_val = True
        for g in self.graphs:
            success = self.test_graph(graph=g, java_mem=java_mem, start_server=start_server)
            if not success:
                ret_val = False
                if break_on_fail:
                    break
        return ret_val

    def build_graph(self, graph_dir, java_mem=None, force_update=False):
        """ will rebuild the graph...
        """
        success = True

        # step 1: set some params
        rebuild_graph = force_update

        # step 2: check graph file is fairly recent and properly sized
        graph_path = os.path.join(graph_dir, self.graph_name)
        if not file_utils.exists_and_sized(graph_path, self.graph_size):
            rebuild_graph = True

        # step 3: check the cache files
        if file_utils.dir_has_newer_files(graph_path, graph_dir, offset_minutes=60, include_filter=".jar,.json,.osm,.zip"):
            rebuild_graph = True
{'fraction_non_alphanumeric': 0.07903347596274855, 'fraction_numerical': 0.004530581424616159, 'mean_word_length': 2.6008155867693703, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5377533', 'n_tokens_mistral': 2409, 'n_tokens_neox': 2308, 'n_words': 717}