<commit_before>import os
import re
import time

import requests
import simplejson as json
from pyquery import PyQuery as pq

USAGE = '%prog trip_id_or_url_1 [trip_id_or_url_2 ...] [--trailauth COOKIE] [options]'
DESCRIPTION = "Scrape EveryTrail trip page(s) and download their contents, including GPX, story and photos. Arguments may be EveryTrail trip IDs (e.g. 2991898) or trip page URLs (e.g. everytrail.com/view_trip.php?trip_id=2991898)."

URL_BASE = 'http://www.everytrail.com'
TRIP_URL_TEMPLATE = '/view_trip.php?trip_id={0}'
TRIP_URL_RE = re.compile('\/view_trip\.php\?trip_id=(\d+)')
GPX_URL_TEMPLATE = '/downloadGPX.php?trip_id={0}'
OUT_DIR = 'trails'

def main():
    """Process arguments when this script is run from the command line."""
    import sys
    from optparse import OptionParser

    parser = OptionParser(usage=USAGE, description=DESCRIPTION)
    parser.add_option('--trailauth',
        action='store', type='string', dest='trailauth', metavar='COOKIE',
        help='the value of the TRAILAUTH cookie from your web browser where you are logged into EveryTrail. This is necessary to enable downloading GPX files. It looks something like "d9b61a...". (See README for help on finding this value.)')
    parser.add_option('--skip-photos',
        action='store_true', dest='skip_photos', default=False,
        help="don't download photos")
    parser.add_option('--out-dir',
        action='store', type='string', dest='out_dir', default=OUT_DIR,
        help="name of output directory where trip data will be saved (default: %default)")
    parser.add_option('--trips-page',
        action='store', type='string', dest='trips_page', metavar='URL',
        help="the URL of a trip listing page which will be scraped for individual trip URLs, e.g. everytrail.com/my_trips.php?user_id=154142. This can be used instead of, or in addition to, specifying trip IDs/URLs as command arguments.")

    options, args = parser.parse_args(sys.argv[1:])

    trip_ids = map(normalize_arg_to_id, args)
    if options.trips_page:
        trip_ids += get_trip_ids_from_listing_page(options.trips_page)

    if not trip_ids:
        parser.print_help()
        sys.exit(0)

    if not options.trailauth:
        print "Will not download GPX files since no TRAILAUTH cookie was provided. `python {0} --help` for more information.".format(os.path.basename(sys.argv[0]))

    for i, trip_id in enumerate(trip_ids):
        print "Trip {0}/{1}:".format(i + 1, len(trip_ids))
        download_trip(trip_id, options.out_dir, trailauth_cookie=options.trailauth, skip_photos=options.skip_photos)

def normalize_arg_to_id(arg):
    if re.match('^\d+$', arg):
        return arg
    else:
        result = TRIP_URL_RE.search(arg)
        if result:
            return result.group(1)
        else:
            raise Exception("Argument is neither trip ID nor trip page URL: {0}".format(arg))

def get_trip_ids_from_listing_page(trips_page_url):
    print "Scraping {0} for trip URLs...".format(trips_page_url)
    trips_page = get_html(trips_page_url)
    trip_ids = sorted([
        normalize_arg_to_id(url)
        for url in
        trips_page.find('a').map(lambda _, a: a.attrib['href'])
        if TRIP_URL_RE.search(url)
    ])
    print "Found links to {0} trips: {1}".format(len(trip_ids), ", ".join(trip_ids))
    return trip_ids

def trip_name_to_directory_name(name):
    """Convert a name like "Mt. Tam: Ridgecrest Blvd to Alpine Lake via..." to
    something like "mt-tam-ridgecrest-blvd-to-alpi".
    """
    out = name.lower()
    out = re.sub('[:/\ ]', '-', out)
    out = re.sub('[.,]', '', out)
    out = re.sub('--+', '-', out)
    out = out[:30]
    return out

def _get_with_retries(url, cookies, max_retries_message, retry_count=0):
    if retry_count >= 3:
        raise Exception(max_retries_message)
    resp = requests.get(url, cookies=cookies)
    if resp.status_code == 200:
        return resp
    else:
        print "----- Response came back with HTTP status {0}; trying again... -----".format(resp.status_code)
        return _get_with_retries(url, cookies, max_retries_message, retry_count=retry_count + 1)

def get_html(url, retry_count=0):
    resp = _get_with_retries(url, cookies=None, max_retries_message="Retried too many times. Maybe EveryTrail is down?")
    return pq(resp.content)

def get_gpx(url, trailauth_cookie, retry_count=0):<commit_msg>Tweaks to help/output text<commit_after>import os
import re
import time

import requests
import simplejson as json
from pyquery import PyQuery as pq

USAGE = '%prog trip_id_or_url_1 [trip_id_or_url_2 ...] [--trailauth COOKIE] [options]'
DESCRIPTION = "Scrape EveryTrail trip page(s) and download their contents, including GPX, story, and photos. Arguments may be EveryTrail trip IDs (e.g. 2991898) or trip page URLs (e.g. http://everytrail.com/view_trip.php?trip_id=2991898)."

URL_BASE = 'http://www.everytrail.com'
TRIP_URL_TEMPLATE = '/view_trip.php?trip_id={0}'
TRIP_URL_RE = re.compile('\/view_trip\.php\?trip_id=(\d+)')
GPX_URL_TEMPLATE = '/downloadGPX.php?trip_id={0}'
OUT_DIR = 'trails'

def main():
    """Process arguments when this script is run from the command line."""
    import sys
    from optparse import OptionParser

    parser = OptionParser(usage=USAGE, description=DESCRIPTION)
    parser.add_option('--trailauth',
        action='store', type='string', dest='trailauth', metavar='COOKIE',
        help='the value of the TRAILAUTH cookie from your web browser where you are logged into EveryTrail. This is necessary to enable downloading GPX files. It looks something like "d9b61a...". (See README for help on finding this value.)')
    parser.add_option('--trips-page',
        action='store', type='string', dest='trips_page', metavar='URL',
        help='the URL of a trip listing page which will be scraped for individual trip URLs, e.g. http://everytrail.com/my_trips.php?user_id=154142. This can be used instead of, or in addition to, specifying trip IDs/URLs as command arguments.')
    parser.add_option('--skip-photos',
        action='store_true', dest='skip_photos', default=False,
        help="don't download photos")
    parser.add_option('--out-dir',
        action='store', type='string', dest='out_dir', default=OUT_DIR,
        help="optionally specify output directory where trip data will be saved (default: %default)")

    options, args = parser.parse_args(sys.argv[1:])

    trip_ids = map(normalize_arg_to_id, args)
    if options.trips_page:
        trip_ids += get_trip_ids_from_listing_page(options.trips_page)

    if not trip_ids:
        parser.print_help()
        sys.exit(0)

    if not options.trailauth:
        print "Will not download GPX files since no TRAILAUTH cookie was provided. `python {0} --help` for more information.".format(os.path.basename(sys.argv[0]))

    for i, trip_id in enumerate(trip_ids):
        print "Trip {0}/{1}:".format(i + 1, len(trip_ids))
        download_trip(trip_id, options.out_dir, trailauth_cookie=options.trailauth, skip_photos=options.skip_photos)

def normalize_arg_to_id(arg):
    if re.match('^\d+$', arg):
        return arg
    else:
        result = TRIP_URL_RE.search(arg)
        if result:
            return result.group(1)
        else:
            raise Exception("Argument is neither trip ID nor trip page URL: {0}".format(arg))

def get_trip_ids_from_listing_page(trips_page_url):
    print "Scraping {0} for trip URLs...".format(trips_page_url)
    trips_page = get_html(trips_page_url)
    trip_ids = sorted([
        normalize_arg_to_id(url)
        for url in
        trips_page.find('a').map(lambda _, a: a.attrib['href'])
        if TRIP_URL_RE.search(url)
    ])
    print "Found links to {0} trips: {1}".format(len(trip_ids), " ".join(trip_ids))
    return trip_ids

def trip_name_to_directory_name(name):
    """Convert a name like "Mt. Tam: Ridgecrest Blvd to Alpine Lake via..." to
    something like "mt-tam-ridgecrest-blvd-to-alpi".
    """
    out = name.lower()
    out = re.sub('[:/\ ]', '-', out)
    out = re.sub('[.,]', '', out)
    out = re.sub('--+', '-', out)
    out = out[:30]
    return out

def _get_with_retries(url, cookies, max_retries_message, retry_count=0):
    if retry_count >= 3:
        raise Exception(max_retries_message)
    resp = requests.get(url, cookies=cookies)
    if resp.status_code == 200:
        return resp
    else:
        print "----- Response came back with HTTP status {0}; trying again... -----".format(resp.status_code)
        return _get_with_retries(url, cookies, max_retries_message, retry_count=retry_count + 1)

def get_html(url, retry_count=0):
    resp = _get_with_retries(url, cookies=None, max_retries_message="Retried too many times. Maybe EveryTrail is down?")
    return pq(resp.content)

def get_gpx(url, trailauth_cookie, retry_count=0):
{'fraction_non_alphanumeric': 0.12775380422439245, 'fraction_numerical': 0.011355893708834885, 'mean_word_length': 3.613410162388685, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1964863', 'n_tokens_mistral': 3037, 'n_tokens_neox': 2902, 'n_words': 772}