<commit_before>#!/usr/bin/env python

"""
Created on 2015-09-26T12:13:49
"""

from __future__ import division, print_function
import sys
import argparse
import re
import time
try:
    import numpy as np
except ImportError:
    print('You need numpy installed')
    sys.exit(1)
import pandas as pd
from splinter.browser import Browser
import connect_aws_db as cadb


__author__ = "<NAME> (github: @mattgiguere)"
__license__ = "MIT"
__version__ = '0.0.1'
__maintainer__ = "<NAME>"
__email__ = "<EMAIL>"
__status__ = " Development NOT(Prototype or Production)"


def get_city(city):
    city_urls = {'new_york_ny': 'http://www.tripadvisor.com/Hotels-g60763-New_York_City_New_York-Hotels.html',
                 'los_angeles_ca': 'http://www.tripadvisor.com/Hotels-g32655-Los_Angeles_California-Hotels.html',
                 'chicago_il': 'http://www.tripadvisor.com/Hotels-g35805-Chicago_Illinois-Hotels.html',
                 'houston_tx': 'http://www.tripadvisor.com/Hotels-g56003-Houston_Texas-Hotels.html',
                 'philadelphia_pa': 'http://www.tripadvisor.com/Hotels-g60795-Philadelphia_Pennsylvania-Hotels.html',
                 'phoenix_az': 'http://www.tripadvisor.com/Hotels-g31310-Phoenix_Arizona-Hotels.html',
                 'san_antonio_tx': 'http://www.tripadvisor.com/Hotels-g60956-San_Antonio_Texas-Hotels.html',
                 'san_diego_ca': 'http://www.tripadvisor.com/Hotels-g60750-San_Diego_California-Hotels.html',
                 'dallas_tx': 'http://www.tripadvisor.com/Hotels-g55711-Dallas_Texas-Hotels.html',
                 'san_jose_ca': 'http://www.tripadvisor.com/Hotels-g33020-San_Jose_California-Hotels.html',
                 'austin_tx': 'http://www.tripadvisor.com/Hotels-g30196-Austin_Texas-Hotels.html',
                 'jacksonville_fl': 'http://www.tripadvisor.com/Hotels-g60805-Jacksonville_Florida-Hotels.html',
                 'san_francisco_ca': 'http://www.tripadvisor.com/Hotels-g60713-San_Francisco_California-Hotels.html',
                 'indianapolis_in': 'http://www.tripadvisor.com/Hotels-g37209-Indianapolis_Indiana-Hotels.html',
                 'columbus_oh': 'http://www.tripadvisor.com/Hotels-g50226-Columbus_Ohio-Hotels.html',
                 'new_haven_ct': 'http://www.tripadvisor.com/Hotels-g33851-New_Haven_Connecticut-Hotels.html',
                 }
    return city_urls[city]


def splinter_scrape_ta(city_url='', city='new_haven', state='ct', write_to_db=False):
    """PURPOSE: To """
    # this only needs to be done at the very beginning
    br = Browser()

    # number of pages of hotel results to scrape
    max_pages = 7

    if city_url == '':
        city_url = get_city(city.lower()+'_'+state.lower())

    print('using the following url:')
    print('{}'.format(city_url))
    #city_url = "http://www.tripadvisor.com/Hotels-g31310-Phoenix_Arizona-Hotels.html"

    #####################################################
    # do not edit below this line
    #####################################################
    # more_pages is used to keep track if there is more
    # than one page of hotel results for the given city
    more_pages = True

    # scraping will start on page 1 of the hotel results
    page = 1

    # open the URL in a browser object:
    br.visit(city_url)

    # find the div to enter the date range. This is needed to get pricing info:
    date_bar = br.find_by_xpath('//*[contains(@class, "meta_date_wrapper")]')

    # find the check in calendar span:
    cin_btn = date_bar.find_by_xpath('span[contains(@class, "meta_date_field   check_in")]/span')[0]

    # now click the check_in span to activate it
    cin_btn.click()<commit_msg>increase max number of pages<commit_after>#!/usr/bin/env python

"""
Created on 2015-09-26T12:13:49
"""

from __future__ import division, print_function
import sys
import argparse
import re
import time
try:
    import numpy as np
except ImportError:
    print('You need numpy installed')
    sys.exit(1)
import pandas as pd
from splinter.browser import Browser
import connect_aws_db as cadb


__author__ = "<NAME> (github: @mattgiguere)"
__license__ = "MIT"
__version__ = '0.0.1'
__maintainer__ = "<NAME>"
__email__ = "<EMAIL>"
__status__ = " Development NOT(Prototype or Production)"


def get_city(city):
    city_urls = {'new_york_ny': 'http://www.tripadvisor.com/Hotels-g60763-New_York_City_New_York-Hotels.html',
                 'los_angeles_ca': 'http://www.tripadvisor.com/Hotels-g32655-Los_Angeles_California-Hotels.html',
                 'chicago_il': 'http://www.tripadvisor.com/Hotels-g35805-Chicago_Illinois-Hotels.html',
                 'houston_tx': 'http://www.tripadvisor.com/Hotels-g56003-Houston_Texas-Hotels.html',
                 'philadelphia_pa': 'http://www.tripadvisor.com/Hotels-g60795-Philadelphia_Pennsylvania-Hotels.html',
                 'phoenix_az': 'http://www.tripadvisor.com/Hotels-g31310-Phoenix_Arizona-Hotels.html',
                 'san_antonio_tx': 'http://www.tripadvisor.com/Hotels-g60956-San_Antonio_Texas-Hotels.html',
                 'san_diego_ca': 'http://www.tripadvisor.com/Hotels-g60750-San_Diego_California-Hotels.html',
                 'dallas_tx': 'http://www.tripadvisor.com/Hotels-g55711-Dallas_Texas-Hotels.html',
                 'san_jose_ca': 'http://www.tripadvisor.com/Hotels-g33020-San_Jose_California-Hotels.html',
                 'austin_tx': 'http://www.tripadvisor.com/Hotels-g30196-Austin_Texas-Hotels.html',
                 'jacksonville_fl': 'http://www.tripadvisor.com/Hotels-g60805-Jacksonville_Florida-Hotels.html',
                 'san_francisco_ca': 'http://www.tripadvisor.com/Hotels-g60713-San_Francisco_California-Hotels.html',
                 'indianapolis_in': 'http://www.tripadvisor.com/Hotels-g37209-Indianapolis_Indiana-Hotels.html',
                 'columbus_oh': 'http://www.tripadvisor.com/Hotels-g50226-Columbus_Ohio-Hotels.html',
                 'new_haven_ct': 'http://www.tripadvisor.com/Hotels-g33851-New_Haven_Connecticut-Hotels.html',
                 }
    return city_urls[city]


def splinter_scrape_ta(city_url='', city='new_haven', state='ct', write_to_db=False, max_pages=20):
    """PURPOSE: To """
    # this only needs to be done at the very beginning
    br = Browser()

    if city_url == '':
        city_url = get_city(city.lower()+'_'+state.lower())

    print('using the following url:')
    print('{}'.format(city_url))
    #city_url = "http://www.tripadvisor.com/Hotels-g31310-Phoenix_Arizona-Hotels.html"

    #####################################################
    # do not edit below this line
    #####################################################
    # more_pages is used to keep track if there is more
    # than one page of hotel results for the given city
    more_pages = True

    # scraping will start on page 1 of the hotel results
    page = 1

    # open the URL in a browser object:
    br.visit(city_url)

    # find the div to enter the date range. This is needed to get pricing info:
    date_bar = br.find_by_xpath('//*[contains(@class, "meta_date_wrapper")]')

    # find the check in calendar span:
    cin_btn = date_bar.find_by_xpath('span[contains(@class, "meta_date_field   check_in")]/span')[0]

    # now click the check_in span to activate it
    cin_btn.click()
{'fraction_non_alphanumeric': 0.15399422521655437, 'fraction_numerical': 0.029561391447820707, 'mean_word_length': 3.3375074537865235, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 34, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4594771', 'n_tokens_mistral': 2669, 'n_tokens_neox': 2409, 'n_words': 436}