<commit_before>from ott.utils import object_utils

from ott.osm.osm_cache import OsmCache
from ott.loader.gtfs.gtfs_cache import GtfsCache
from ott.loader.otp.graph.otp_exporter import OtpExporter
from ott.loader.otp.graph.otp_builder import OtpBuilder
from ott.loader.gtfsdb.gtfsdb_loader import GtfsdbLoader
from ott.loader.gtfsdb.gtfsdb_exporter import GtfsdbExporter
from ott.loader.gtfsdb_realtime.gtfsdb_realtime_loader import GtfsdbRealtimeLoader
from ott.loader.sum.sum_cache import SumCache
from ott.loader.solr.solr_loader import SolrLoader

import logging
log = logging.getLogger(__file__)


def download_data():
    """ just download data

        does the following:
          1.  update GTFS feeds in cache
          2.  update OSM data in cache
          3.  update SUM data in cache
    """
    force_update=object_utils.is_force_update()

    log.info("step 1: cache latest gtfs feeds")
    gtfs = GtfsCache()
    updated_feeds = gtfs.check_cached_feeds(force_update=force_update)
    # NOTE: to do item could be to check updated_feeds (list of feed names) for certain agency names, and selectively force update
    if updated_feeds and len(updated_feeds) > 0:
        force_update = True

    log.info("step 2: cache latest osm data")
    updated_osm = OsmCache.update(force_update=force_update)
    if updated_osm:
        force_update = True

    log.info("step 3: download SUM data (BIKETOWN)")
    sum_update = SumCache.load

    return force_update


def load_all():
    """ will load OTP and gtfsdb

          load gtfsdb
          pg_dump an export file for gtfsdb

          build OTP graph
          test OTP graph

          load SOLR with cached data
    """
    force_update=object_utils.is_force_update()

    # steps 1 - 3: download data (see above)
    # import pdb; pdb.set_trace()
    download_data()

    log.info("step 4: load gtfsdb")
    db = GtfsdbLoader()
    db.check_db(force_update=force_update)

    log.info("step 5: load gtfsdb_realtime db")
    rt = GtfsdbRealtimeLoader()
    rt.load_all(is_geospatial=True, create_db=True)

    log.info("step 6: load otp (build new graph)")
    #otp = OtpBuilder(force_update=force_update)
    #otp.build_and_test_graphs(force_update=force_update)

    log.info("step 7: load various data layers into SOLR")
    solr_load = SolrLoader.load()<commit_msg>clean up some commented out routines<commit_after>from ott.utils import object_utils

from ott.osm.osm_cache import OsmCache
from ott.loader.gtfs.gtfs_cache import GtfsCache
from ott.loader.otp.graph.otp_exporter import OtpExporter
from ott.loader.otp.graph.otp_builder import OtpBuilder
from ott.loader.gtfsdb.gtfsdb_loader import GtfsdbLoader
from ott.loader.gtfsdb.gtfsdb_exporter import GtfsdbExporter
from ott.loader.gtfsdb_realtime.gtfsdb_realtime_loader import GtfsdbRealtimeLoader
from ott.loader.sum.sum_cache import SumCache
from ott.loader.solr.solr_loader import SolrLoader

import logging
log = logging.getLogger(__file__)


def download_data():
    """ just download data

        does the following:
          1.  update GTFS feeds in cache
          2.  update OSM data in cache
          3.  update SUM data in cache
    """
    force_update=object_utils.is_force_update()

    log.info("step 1: cache latest gtfs feeds")
    gtfs = GtfsCache()
    updated_feeds = gtfs.check_cached_feeds(force_update=force_update)
    # NOTE: to do item could be to check updated_feeds (list of feed names) for certain agency names, and selectively force update
    if updated_feeds and len(updated_feeds) > 0:
        force_update = True

    log.info("step 2: cache latest osm data")
    updated_osm = False
    updated_osm = OsmCache.update(force_update=force_update)
    if updated_osm:
        force_update = True

    log.info("step 3: download SUM data (BIKETOWN)")
    sum_update = SumCache.load

    return force_update


def load_all():
    """ will load OTP and gtfsdb

          load gtfsdb
          pg_dump an export file for gtfsdb

          build OTP graph
          test OTP graph

          load SOLR with cached data
    """
    force_update=object_utils.is_force_update()

    # steps 1 - 3: download data (see above)
    # import pdb; pdb.set_trace()
    download_data()

    log.info("step 4: load gtfsdb")
    db = GtfsdbLoader()
    db.check_db(force_update=force_update)

    log.info("step 5: load gtfsdb_realtime db")
    rt = GtfsdbRealtimeLoader()
    rt.load_all(is_geospatial=True, create_db=True)

    log.info("step 6: load otp (build new graph)")
    otp = OtpBuilder(force_update=force_update)
    otp.build_and_test_graphs(force_update=force_update)

    log.info("step 7: load various data layers into SOLR")
    solr_load = SolrLoader.load()
{'fraction_non_alphanumeric': 0.07649925831744013, 'fraction_numerical': 0.005509641873278237, 'mean_word_length': 3.440263405456256, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '470908', 'n_tokens_mistral': 1617, 'n_tokens_neox': 1561, 'n_words': 416}