<commit_before>namespace :scraper do
	desc "fetch posts from 3Taps for CraigsList since the last anchor"
	task scrape: :environment do
	require 'open-uri'
	require 'json'
	auth_token = "1b9df10b526e6f785dfdc940a414ffae"
	polling_url = "http://polling.3taps.com/poll"
	hipster_locs = ["USA-MIA-MIB", "USA-MIA-FOR", "USA-MIA-MIF", "USA-MIA-DEE", "USA-MIA-DOM"]
	#Grab data until up to date
	loop do 

		# Specify request params
		# To get a new anchor change the date in the url to a unix time for todays date(without time)
		# http://polling.3taps.com/anchor?auth_token=1b9df10b526e6f785dfdc940a414ffae&timestamp=1432598400
		params = {
			auth_token: auth_token,
			anchor:  Anchor.first.value, 
			source: "CRAIG",
			category_group: "RRRR",
			category: "RHFR",
			'location.city' => "USA-MIA-MIF",
			retvals: "location,external_url,heading,body,timestamp,price,images,annotations"
		}


		# Prepare API Request
		uri = URI.parse(polling_url)
		uri.query = URI.encode_www_form(params)<commit_msg>added some hipster locations to the filter. Need to make the filter dynamic<commit_after>namespace :scraper do
	desc "fetch posts from 3Taps for CraigsList since the last anchor"
	task scrape: :environment do
	require 'open-uri'
	require 'json'
	auth_token = "1b9df10b526e6f785dfdc940a414ffae"
	polling_url = "http://polling.3taps.com/poll"
	hipster_locs = ["USA-MIA-MIB", "USA-MIA-FOR", "USA-MIA-MIF", "USA-MIA-DEE", "USA-MIA-DOM", "USA-MIA-NOC", "USA-MIA-SOC", "USA-MIA-WYN"]
	#Grab data until up to date
	loop do 

		# Specify request params
		# To get a new anchor change the date in the url to a unix time for todays date(without time)
		# http://polling.3taps.com/anchor?auth_token=<PASSWORD>&timestamp=<PASSWORD>
		params = {
			auth_token: auth_token,
			anchor:  Anchor.first.value, 
			source: "CRAIG",
			category_group: "RRRR",
			category: "RHFR",
			'location.city' => "USA-MIA-MIF",
			retvals: "location,external_url,heading,body,timestamp,price,images,annotations"
		}


		# Prepare API Request
		uri = URI.parse(polling_url)
		uri.query = URI.encode_www_form(params)
{'fraction_non_alphanumeric': 0.12385981757081133, 'fraction_numerical': 0.03216514642342775, 'mean_word_length': 4.4986807387862795, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1840009', 'n_tokens_mistral': 879, 'n_tokens_neox': 786, 'n_words': 184}