<commit_before># _*_ coding:utf-8 _*_
from __future__ import division
from sklearn import tree
import math
from textblob import TextBlob as tb

import mechanize
import cookielib
from BeautifulSoup import BeautifulSoup
import html2text
import scrapy 
import urllib2
from tidylib import tidy_document
import re

import blockspring
import json
import unidecode
import requests


# #0 for bumpy, 1 for smooth
# features = [[140, 1], [130, 1], [150, 0], [170, 0]]
# #0 for apple, 1 for orange
# labels = [0, 0, 1, 1]

# classifier= tree.DecisionTreeClassifier()

# classifier= classifier.fit(features,labels)

# print classifier.predict([[160,0]])


#term frequency is the number of times a word appears in a text file (textfile), 
#normalized (averaged) by dividing by the total number of words in that file. 

#TextBlob will break the textfile into words and get the word counts
def tf(word, textfile):
    return textfile.words.count(word) / len(textfile.words)

#returns the number of documents containing word. 
#A generator expression is passed to the sum() function.

#text files is a list of text files
def n_containing(word, textfiles):
    return sum(1 for textfile in textfiles if word in textfile.words)

#computes "inverse document frequency" which measures 
#how common a word is among all documents in textfile_list. 
#The more common a word is, the lower its idf. 

#We take the ratio of the total number of documents 
#to the number of documents containing word, then take the log of that. 

#Add 1 to the divisor to prevent division by zero.

def idf(word, textfiles ):
    #bottom = (1 + n_containing(word, textfiles))
    # print "bottom: {}".format(bottom)

    #top = len(textfiles)
    # print "top: {}".format(top)

    #num = math.log((3)/4)
    # print "log: {}".format(num)

    result = math.log((len(textfiles)) / (1 + n_containing(word, textfiles)))
    # print "result: {}".format(result)
    return result

#computes the TF-IDF score. It is simply the product of  tf and idf.
def tfidf(word, texfile, textfiles):
    return tf(word, texfile) * idf(word, textfiles)

########
# Request to Indeed API for job ad list should go here

indeed_request_url="http://api.indeed.com/ads/apisearch?publisher=9253729351823762&q=software engineer&l=seattle%2C+wa&sort=&radius=&st=&jt=&start={}&limit=1000&fromage=&filter=&latlong=1&co=us&chnl=&userip=1.2.3.4&useragent=Mozilla/%2F4.0%28Firefox%29&v=2&format=json"


def get_job_urls_batch(request_url):
    response = requests.get(request_url)
    json_data=response.json()
    job_ad_urls=[]
    # print type(json_data["results"])
    for i, result in enumerate(json_data["results"]):
        # print result
        this_url= result["url"]
        job_ad_urls.append(this_url)  
        # print this_url
    return job_ad_urls

def get_mult_batches(request_url,num_records):
    start = 0
    batches =[]
    for i in range(((num_records-1)//25)+1):
       request_url=request_url.format(start)
       this_batch=get_job_urls_batch(request_url)
       batches.extend(this_batch)
       start+=25
    return batches

# get_job_urls_batch(indeed_request_url)
these_batches = get_mult_batches(indeed_request_url,100)

print "100 job urls: {}".format(these_batches)
print "job url 1: {}".format(these_batches[0])
print "job url 2: {}".format(these_batches[1])
print "job url 3: {}".format(these_batches[2])
# indeed_request_url+=.format(num_times)


# print "Number of results: {}".format(len(response.json()["results"]))

# Get url of a job ad from Indeed API response


#########

# get get text from job ad url
job_ad_url = these_batches[0]
#test_url = "https://jobs.lever.co/palantir/7a2cf40e-ab26-4b14-ae1d-0919625816ce"
html = urllib2.urlopen(job_ad_url).read()



# tidy, errors = tidy_document(html)
# # soup = BeautifulSoup(tidy)
soup = BeautifulSoup(html)

job_ad_text= soup
# texts = soup.findAll(text=True)

# def visible(element):
#     if element.parent.name in ['style', 'script', '[document]', 'head', 'title']:
#         return False
#     elif re.match('<!--.*-->', str(element)):
#         return False
#     return True

# visible_texts = filter(visible, texts)
job_ad = ' '.join(job_ad_text)

print job_ad

job_ad_nouns = tb(job_ad).noun_phrases
print "Nouns:{}".format(job_ad_nouns) 



<commit_msg>fixes the guts to match master branch<commit_after># _*_ coding:utf-8 _*_
from __future__ import division
from sklearn import tree
import math
from textblob import TextBlob as tb

import mechanize
import cookielib
from BeautifulSoup import BeautifulSoup
import html2text
import scrapy 
import urllib2
from tidylib import tidy_document
import re

import blockspring
import json
import unidecode
import requests


# #0 for bumpy, 1 for smooth
# features = [[140, 1], [130, 1], [150, 0], [170, 0]]
# #0 for apple, 1 for orange
# labels = [0, 0, 1, 1]

# classifier= tree.DecisionTreeClassifier()

# classifier= classifier.fit(features,labels)

# print classifier.predict([[160,0]])


#term frequency is the number of times a word appears in a text file (textfile), 
#normalized (averaged) by dividing by the total number of words in that file. 

#TextBlob will break the textfile into words and get the word counts
def tf(word, textfile):
    return textfile.words.count(word) / len(textfile.words)

#returns the number of documents containing word. 
#A generator expression is passed to the sum() function.

#text files is a list of text files
# number of text files that have these word
def n_containing(word, textfiles):
    return sum(1 for textfile in textfiles if word in textfile.words)

# take most common queries (software engineer, data scientist,etc. ) and pre compute keyword frequency to reduce runtime
# replace above with..

#computes "inverse document frequency" which measures 
#how common a word is among all documents in textfile_list. 
#The more common a word is, the lower its idf. 

#We take the ratio of the total number of documents 
#to the number of documents containing word, then take the log of that. 

#Add 1 to the divisor to prevent division by zero.

def idf(word, textfiles ):
    #bottom = (1 + n_containing(word, textfiles))
    # print "bottom: {}".format(bottom)

    #top = len(textfiles)
    # print "top: {}".format(top)

    #num = math.log((3)/4)
    # print "log: {}".format(num)

    result = math.log((len(textfiles)) / (1 + n_containing(word, textfiles)))
    # print "result: {}".format(result)
    return result

#computes the TF-IDF score. It is simply the product of  tf and idf.
def tfidf(word, texfile, textfiles):
    return tf(word, texfile) * idf(word, textfiles)

########
# Request to Indeed API for job ad list should go here

indeed_request_url="http://api.indeed.com/ads/apisearch?publisher=9253729351823762&q=software engineer&l=seattle%2C+wa&sort=&radius=&st=&jt=&start={}&limit=1000&fromage=&filter=&latlong=1&co=us&chnl=&userip=1.2.3.4&useragent=Mozilla/%2F4.0%28Firefox%29&v=2&format=json"


def get_job_urls_batch(request_url):
    response = requests.get(request_url)
    json_data=response.json()
    job_ad_urls=[]
    # print type(json_data["results"])
    for i, result in enumerate(json_data["results"]):
        # print result
        this_url= result["url"]
        job_ad_urls.append(this_url)  
        # print this_url
    return job_ad_urls

def get_mult_batches(request_url,num_records):
    start = 0
    batches =[]
    for i in range(((num_records-1)//25)+1):
       request_url=request_url.format(start)
       this_batch=get_job_urls_batch(request_url)
       batches.extend(this_batch)
       start+=25
    return batches

# get_job_urls_batch(indeed_request_url)
these_batches = get_mult_batches(indeed_request_url,100)

print "100 job urls: {}".format(these_batches)

# indeed_request_url+=.format(num_times)


# print "Number of results: {}".format(len(response.json()["results"]))

# Get url of a job ad from Indeed API response


#########

# get get text from job ad url
job_ad_url = these_batches[0]
print "First job ad url: {}".format(job_ad_url)
#test_url = "https://jobs.lever.co/palantir/7a2cf40e-ab26-4b14-ae1d-0919625816ce"
html = urllib2.urlopen(job_ad_url).read()



# tidy, errors = tidy_document(html)
# # soup = BeautifulSoup(tidy)
soup = BeautifulSoup(html)
job_ad_text = soup.find('span', attrs={'id': 'job_summary'}).text

print "hellooooooooooooooooooooo"
print job_ad_text
print "hellooooooooooooooooooooo"
print type(job_ad_text)


# print job_ad

job_ad_nouns = tb(job_ad_text).noun_phrases
print "Nouns:{}".format(job_ad_nouns) 




{'fraction_non_alphanumeric': 0.1186202041534671, 'fraction_numerical': 0.02557784817552505, 'mean_word_length': 4.524303305249514, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1176643', 'n_tokens_mistral': 3057, 'n_tokens_neox': 2880, 'n_words': 838}