<commit_before>######################################
# File Name : indeed_scrape.py
# Author : <NAME>
# Creation Date : 11-05-2015
######################################
#TODO: add logger
#TODO: use MRJob mapper for parsing
import ConfigParser
import json
import pandas as pd
import urllib2
from bs4 import BeautifulSoup
from sklearn.feature_extraction.text import CountVectorizer, ENGLISH_STOP_WORDS
from sklearn.metrics.pairwise import cosine_distances
from sklearn.cluster import KMeans
from sklearn.metrics import euclidean_distances
import matplotlib.pyplot as plt
import numpy as np
from nltk import stem
from nltk import tokenize
import re

toker = tokenize.word_tokenize
stemmer = stem.SnowballStemmer('english')


class Indeed(object):
    def __init__(self):
        self.add_loc = None
        self.stop_words = None
        self.num_samp = 300
        self.zip_code_file ='us_postal_codes.csv'
        self.df = pd.DataFrame(columns=['url'])
        self.config_path = "tokens.cfg"
        self.query = None
        self.locations = None

    def add_stop_words(self):
        if self.stop_words is not None:
            words = self.stop_words.split(" ")
            self.stop_words = ENGLISH_STOP_WORDS.union(words)

    def build_api_string(self):
        if self.query is None:
            print "query cannot be empty"
            raise ValueError

        # beware of escaped %
        query = self.format_query(self.query)
        prefix = 'http://api.indeed.com/ads/apisearch?'
        pub = 'publisher=%(pub_id)s'
        chan = '&chnl=%(channel_name)s'
        loc = '&l=%(loc)s'
        query = '&q=%s' % query
        start = '&start=0'
        frm = '&fromage=30'
        limit = '&limit=25'
        site = '&st=employer'
        format = '&format=json'
        sort = '&sort=0'
        country = '&co=us'
        suffix = '&userip=1.2.3.4&useragent=Mozilla/%%2F4.0%%28Firefox%%29&v=2'

        self.api = prefix + pub + chan + loc + query + start + frm + limit + \
                   site + format + country + sort + suffix

    def format_query(self, query):
         return "%%20".join(query.split(" "))

    def load_config(self):
        '''loads a config file that contains tokens'''
        config_parser = ConfigParser.RawConfigParser()
        config_parser.read(self.config_path)
        self.pub_id = config_parser.get("id", "pub_id")
        self.channel_name = config_parser.get("channel", "channel_name")

    def load_zipcodes(self):
        '''loads the zip code file and returnes a list of zip codes'''
        self.df_zip = pd.read_csv(self.zip_code_file, dtype=str)
        self.df_zip.dropna(inplace=True, how='all')
        locations = self.df_zip['Postal Code'].dropna(how='any')

        return locations

    def get_city_url_content_stem(self):
        ind = 0
        for zipcode in self.locations:
            url_city = self.get_url(zipcode)
            if url_city is not None:
                for item in url_city:
                    if item[0] not in self.df['url']:
                        self.df.loc[ind, 'zipcode'] = str(zipcode)
                        self.df.loc[ind, 'url'] = item[0]
                        self.df.loc[ind, 'city'] = item[1]
                        content = self.parse_content(item[0])
                        self.df.loc[ind, 'summary'] = content
                        self.df.loc[ind, 'summary_stem'] = self.stemmer_(content)
                        ind += 1

    def get_urls(self):
        urls = []
        for item in self.locations:
            url = self.get_url(item)
            if url is not None:
                urls.extend(url)

        return urls

    def get_url(self, location):
        '''method good for use with MapReduce'''

        api = self.api %{'pub_id':self.pub_id,
                         'loc':location,
                         'channel_name':self.channel_name
                        }

        try:
            response = urllib2.urlopen(api)
            data = json.loads(response.read())
            response.close()

            urls = []
            urls.extend([ (item['url'], item['city']) for item in data['results']])

        except urllib2.HTTPError, err:
            print err
            return None

        except Exception, err:
            print err
            return None

        return urls

    def get_content(self, url):
        if url is None:
            return None

        try:
            response = urllib2.urlopen(url)
            content = response.read()
            response.close()

            return content

        except urllib2.HTTPError, err:
            print err
            return None

        except Exception, err:
            print err
            return None

    def len_tester(self, word_list):
        new_list = []
        for word in word_list:
            if len(word) < 3:
                continue
            else:
                new_list.append(word)

        return new_list

    def stemmer_(self, string):

        if string is None:
            return None

        words = toker(string)
        words = self.len_tester(words)
        words = map(stemmer.stem, words)

        return " ".join(words)

    def parse_content(self, url):
        content = self.get_content(url)

        if content is None:
            return None

        content = content.decode("ascii", "ignore")
        soup = BeautifulSoup(content, 'lxml')

        try:
            summary = soup.find('span', {'summary'})

        except AttributeError:
            summary = soup.find_all('span')

        if summary is None:
            return None

        bullets = summary.find_all("li")

        if bullets is not None and len(bullets) > 2:
            skills = bullets
        else:
            skills = summary

        try:
            output = [item.get_text() for item in skills]<commit_msg>changed parser<commit_after>######################################
# File Name : indeed_scrape.py
# Author : <NAME>
# Creation Date : 11-05-2015
######################################
#TODO: add logger
#TODO: use MRJob mapper for parsing
import ConfigParser
import json
import pandas as pd
import urllib2
from bs4 import BeautifulSoup
from sklearn.feature_extraction.text import CountVectorizer, ENGLISH_STOP_WORDS
from sklearn.metrics.pairwise import cosine_distances
from sklearn.cluster import KMeans
from sklearn.metrics import euclidean_distances
import matplotlib.pyplot as plt
import numpy as np
from nltk import stem
from nltk import tokenize
import re

toker = tokenize.word_tokenize
stemmer = stem.SnowballStemmer('english')


class Indeed(object):
    def __init__(self):
        self.add_loc = None
        self.stop_words = None
        self.num_samp = 300
        self.zip_code_file ='us_postal_codes.csv'
        self.df = pd.DataFrame(columns=['url'])
        self.config_path = "tokens.cfg"
        self.query = None
        self.locations = None

    def add_stop_words(self):
        if self.stop_words is not None:
            words = self.stop_words.split(" ")
            self.stop_words = ENGLISH_STOP_WORDS.union(words)

    def build_api_string(self):
        if self.query is None:
            print "query cannot be empty"
            raise ValueError

        # beware of escaped %
        query = self.format_query(self.query)
        prefix = 'http://api.indeed.com/ads/apisearch?'
        pub = 'publisher=%(pub_id)s'
        chan = '&chnl=%(channel_name)s'
        loc = '&l=%(loc)s'
        query = '&q=%s' % query
        start = '&start=0'
        frm = '&fromage=30'
        limit = '&limit=25'
        site = '&st=employer'
        format = '&format=json'
        sort = '&sort=0'
        country = '&co=us'
        suffix = '&userip=1.2.3.4&useragent=Mozilla/%%2F4.0%%28Firefox%%29&v=2'

        self.api = prefix + pub + chan + loc + query + start + frm + limit + \
                   site + format + country + sort + suffix

    def format_query(self, query):
         return "%%20".join(query.split(" "))

    def load_config(self):
        '''loads a config file that contains tokens'''
        config_parser = ConfigParser.RawConfigParser()
        config_parser.read(self.config_path)
        self.pub_id = config_parser.get("id", "pub_id")
        self.channel_name = config_parser.get("channel", "channel_name")

    def load_zipcodes(self):
        '''loads the zip code file and returnes a list of zip codes'''
        self.df_zip = pd.read_csv(self.zip_code_file, dtype=str)
        self.df_zip.dropna(inplace=True, how='all')
        locations = self.df_zip['Postal Code'].dropna(how='any')

        return locations

    def get_city_url_content_stem(self):
        ind = 0
        for zipcode in self.locations:
            url_city = self.get_url(zipcode)
            if url_city is not None:
                for item in url_city:
                    if item[0] not in self.df['url']:
                        self.df.loc[ind, 'zipcode'] = str(zipcode)
                        self.df.loc[ind, 'url'] = item[0]
                        self.df.loc[ind, 'city'] = item[1]
                        content = self.parse_content(item[0])
                        self.df.loc[ind, 'summary'] = content
                        self.df.loc[ind, 'summary_stem'] = self.stemmer_(content)
                        ind += 1

    def get_urls(self):
        urls = []
        for item in self.locations:
            url = self.get_url(item)
            if url is not None:
                urls.extend(url)

        return urls

    def get_url(self, location):
        '''method good for use with MapReduce'''

        api = self.api %{'pub_id':self.pub_id,
                         'loc':location,
                         'channel_name':self.channel_name
                        }

        try:
            response = urllib2.urlopen(api)
            data = json.loads(response.read())
            response.close()

            urls = []
            urls.extend([ (item['url'], item['city']) for item in data['results']])

        except urllib2.HTTPError, err:
            print err
            return None

        except Exception, err:
            print err
            return None

        return urls

    def get_content(self, url):
        if url is None:
            return None

        try:
            response = urllib2.urlopen(url)
            content = response.read()
            response.close()

            return content

        except urllib2.HTTPError, err:
            print err
            return None

        except Exception, err:
            print err
            return None

    def len_tester(self, word_list):
        new_list = []
        for word in word_list:
            if len(word) < 3:
                continue
            else:
                new_list.append(word)

        return new_list

    def stemmer_(self, string):

        if string is None:
            return None

        words = toker(string)
        words = self.len_tester(words)
        words = map(stemmer.stem, words)

        return " ".join(words)

    def parse_content(self, url):
        content = self.get_content(url)

        if content is None:
            return None

        content = content.decode("ascii", "ignore")
        soup = BeautifulSoup(content, 'html.parse')

        try:
            summary = soup.find('span', {'summary'})

        except AttributeError:
            summary = soup.find_all('span')

        if summary is None:
            return None

        bullets = summary.find_all("li")

        if bullets is not None and len(bullets) > 2:
            skills = bullets
        else:
            skills = summary

        try:
            output = [item.get_text() for item in skills]
{'fraction_non_alphanumeric': 0.1138592750533049, 'fraction_numerical': 0.00767590618336887, 'mean_word_length': 1.8412890719651078, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '572048', 'n_tokens_mistral': 3556, 'n_tokens_neox': 3381, 'n_words': 862}