<commit_before>from ott.utils import object_utils

from ott.loader.gtfs.gtfs_cache import GtfsCache
from ott.loader.osm.osm_cache import OsmCache
from ott.loader.otp.graph.otp_exporter import OtpExporter
from ott.loader.otp.graph.otp_builder import OtpBuilder
from ott.loader.gtfsdb.gtfsdb_loader import GtfsdbLoader
from ott.loader.sum.sum_cache import SumCache
from ott.loader.solr.solr_loader import SolrLoader

import logging
# logging.basicConfig(level=logging.INFO)
log = logging.getLogger(__file__)


def load_data():
    """ just download data

        does the following:
          1.  update GTFS feeds in cache
          2.  update OSM data in cache
          3.  update SUM data in cache
    """
    force_update=object_utils.is_force_update()

    log.info("step 1: cache latest gtfs feeds")
    gtfs = GtfsCache()
    gtfs.check_cached_feeds(force_update=force_update)

    log.info("step 2: cache latest osm data")
    osm = OsmCache()
    osm.check_cached_osm(force_update=force_update)

    log.info("step 3: download SUM data (BIKETOWN)")
    sum_update = SumCache.load


def load_all():
    """ will load OTP and gtfsdb

          3a. load gtfsdb
          3b. export gtfsdb to production

          4a. build OTP graph
          4b. test OTP graph
          4c. deploy graphs that pass tests to production servers

          5. load SOLR with cached datad
    """
    force_update=object_utils.is_force_update()

    load_data()

    log.info("step 4: load gtfsdb")
    db = GtfsdbLoader()
    db.check_db(force_update=force_update)

    log.info("step 5: load otp (build new graph)")
    otp = OtpBuilder(force_update=force_update)
    otp.build_and_test_graphs(force_update=force_update)

    log.info("step 6: load various data layers into SOLR")
    solr_load = SolrLoader.load()

    export_all()


def export_all():
    """
    """
    log.info("step 1: export gtfsdb ")
    GtfsdbLoader.dump()

    log.info("step 2: load any new OSM database snapshot")
    osm = OsmCache()
    #osm.deploy_db_snapshot()

    log.info("step 3: export otp graph")
    OtpExporter.export()

    log.info("step 4: export... SOLR updates")
    #solr_load = SolrLoader.load


def deploy_all():
    """ load (production) new database extracts and deploy new otp graphs

        does the following:
          1. will  any postgres exports into local (production) db
          2. enable new OTP graph

    """
    log.info("step 1: restore gtfsdb from export")
    GtfsdbLoader.restore()

    log.info("step 2: load any new OSM database snapshot")
    osm = OsmCache()
    #osm.deploy_db_snapshot()

    log.info("step 3: export otp graph")
    OtpExporter.export()

    log.info("step 4: export... SOLR updates")
    #solr_load = SolrLoader.load


def load_and_export():
    log.info("***load and build things, then export them and scp' them to production servers ***")
    load_all()
    export_all()<commit_msg>refactor ... more rename deploy to export<commit_after>from ott.utils import object_utils

from ott.loader.gtfs.gtfs_cache import GtfsCache
from ott.loader.osm.osm_cache import OsmCache
from ott.loader.otp.graph.otp_exporter import OtpExporter
from ott.loader.otp.graph.otp_builder import OtpBuilder
from ott.loader.gtfsdb.gtfsdb_loader import GtfsdbLoader
from ott.loader.sum.sum_cache import SumCache
from ott.loader.solr.solr_loader import SolrLoader

import logging
# logging.basicConfig(level=logging.INFO)
log = logging.getLogger(__file__)


def load_data():
    """ just download data

        does the following:
          1.  update GTFS feeds in cache
          2.  update OSM data in cache
          3.  update SUM data in cache
    """
    force_update=object_utils.is_force_update()

    log.info("step 1: cache latest gtfs feeds")
    gtfs = GtfsCache()
    gtfs.check_cached_feeds(force_update=force_update)

    log.info("step 2: cache latest osm data")
    osm = OsmCache()
    osm.check_cached_osm(force_update=force_update)

    log.info("step 3: download SUM data (BIKETOWN)")
    sum_update = SumCache.load


def load_all():
    """ will load OTP and gtfsdb

          3a. load gtfsdb
          3b. export gtfsdb to production

          4a. build OTP graph
          4b. test OTP graph
          4c. deploy graphs that pass tests to production servers

          5. load SOLR with cached datad
    """
    force_update=object_utils.is_force_update()

    load_data()

    log.info("step 4: load gtfsdb")
    db = GtfsdbLoader()
    db.check_db(force_update=force_update)

    log.info("step 5: load otp (build new graph)")
    otp = OtpBuilder(force_update=force_update)
    otp.build_and_test_graphs(force_update=force_update)

    log.info("step 6: load various data layers into SOLR")
    solr_load = SolrLoader.load()

    export_all()


def export_all():
    """
    """
    log.info("step 1: export gtfsdb ")
    GtfsdbLoader.dump()

    log.info("step 2: load any new OSM database snapshot")
    osm = OsmCache()
    #osm.deploy_db_snapshot()

    log.info("step 3: export otp graph")
    OtpExporter.export()

    log.info("step 4: export... SOLR updates")
    #solr_load = SolrLoader.load


def restore_production():
    """ load (production) new database extracts and deploy new otp graphs

        does the following:
          1. will  any postgres exports into local (production) db
          2. enable new OTP graph

    """
    log.info("step 1: restore gtfsdb from export")
    GtfsdbLoader.restore()

    log.info("step 2: load any new OSM database snapshot")
    osm = OsmCache()
    #osm.restore_db_snapshots()

    log.info("step 4: export... SOLR updates")
    #solr_load = SolrLoader.load

def load_and_export():
    log.info("***load and build things, then export them and scp' them to production servers ***")
    load_all()
    export_all()
{'fraction_non_alphanumeric': 0.09648518263266713, 'fraction_numerical': 0.008442453480358374, 'mean_word_length': 3.05377094972067, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6051890', 'n_tokens_mistral': 1985, 'n_tokens_neox': 1896, 'n_words': 530}