<commit_before># -*- coding: UTF-8 -*-

require 'cgi'
require 'uri'
require 'json'
require 'web_analytics_discovery/grabberutils'

class Quantcast
	include GrabberUtils

	def run(url)
		uri = URI.parse(url)
		host = uri.host

		r = {}

		# Get auth cookies
		doc = download("https://www.quantcast.com/#{host}")
		if doc =~ /<td class="reach" id="reach-(.*?)">/
			r[:id] = id = $1
		else
			return nil
		end

		# Quantcast has no traffic info? We should stop here
		return r if doc =~ /content="We do not have enough information to provide a traffic estimate./

		# Use auth cookies with API call
		id_encoded = CGI::escape(id)
		doc = download(
			"https://www.quantcast.com/api/profile/traffic/?&wUnit=#{id_encoded}&country=US&period=DAY30&countType=",
			'UTF-8',
			'Referer' => 'https://www.quantcast.com/#{host}',
			'X-Requested-With' => 'XMLHttpRequest',
		)

		d = JSON.load(doc).first
#		points = d['reach']['US']['PEOPLE']['WEB']
#		points.each { |pt|
#			puts Time.at(pt['timestamp']).to_s + "\t" + pt['reach'].to_s
#		}

		r[:visitors_mon] = d['summaries']['US']['PEOPLE']['WEB']['reach'].to_i

		return r
	end
end
<commit_msg>Fixed referer for Quantcast (although it doesn't seem to matter)<commit_after># -*- coding: UTF-8 -*-

require 'cgi'
require 'uri'
require 'json'
require 'web_analytics_discovery/grabberutils'

class Quantcast
	include GrabberUtils

	def run(url)
		uri = URI.parse(url)
		host = uri.host

		r = {}

		# Get auth cookies
		doc = download("https://www.quantcast.com/#{host}")
		if doc =~ /<td class="reach" id="reach-(.*?)">/
			r[:id] = id = $1
		else
			return nil
		end

		# Quantcast has no traffic info? We should stop here
		return r if doc =~ /content="We do not have enough information to provide a traffic estimate./

		# Use auth cookies with API call
		id_encoded = CGI::escape(id)
		doc = download(
			"https://www.quantcast.com/api/profile/traffic/?&wUnit=#{id_encoded}&country=US&period=DAY30&countType=",
			'UTF-8',
			'Referer' => "https://www.quantcast.com/#{host}",
			'X-Requested-With' => 'XMLHttpRequest',
		)

		d = JSON.load(doc).first
#		points = d['reach']['US']['PEOPLE']['WEB']
#		points.each { |pt|
#			puts Time.at(pt['timestamp']).to_s + "\t" + pt['reach'].to_s
#		}

		r[:visitors_mon] = d['summaries']['US']['PEOPLE']['WEB']['reach'].to_i

		return r
	end
end

{'fraction_non_alphanumeric': 0.1866151866151866, 'fraction_numerical': 0.00429000429000429, 'mean_word_length': 3.778688524590164, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 6, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6639678', 'n_tokens_mistral': 946, 'n_tokens_neox': 856, 'n_words': 199}