<commit_before>from model.appinfo import AppInfo, AppInfoPlatforms

from urllib2 import urlopen, URLError
from urllib import urlencode, quote
from HTMLParser import HTMLParser
from xml.etree import ElementTree
import re

import json
import logging

class StoreTypeEnum:
	PLAY_STORE = "PlayStore"
	APP_STORE = "AppStore"

class CountryEnum:
	US = 'us'

class PlayStoreCollectionEnum:
	TOP_SELLING_FREE = "topselling_free"
	TOP_SELLING_PAID = "topselling_paid"
	TOP_GROSSING = "topgrossing"
	TOP_SELLING_NEW_FREE = "topselling_new_free"
	TOP_SELLING_NEW_PAID = "topselling_new_paid"

class AppStoreRSSTypeEnum:
	TOP_FREE = 'topfreeapplications'
	NEW_FREE = 'newfreeapplications'
	NEW_PAID = 'newpaidapplications'
	NEW_APPS = 'newapplications'
	TOP_GROSSING = 'topgrossingapplications'
	TOP_PAID = 'toppaidapplications'

class AppStoreRSSGenreEnum:
	ALL = None
	BOOKS = 6018
	BUSINESS = 6000
	CATALOGS = 6022
	EDUCATION = 6017
	ENTERTAINMENT = 6016
	FINANCE = 6015
	FOOD_AND_DRINK = 6023
	GAMES = 6014
	HEALTH_AND_FITNESS = 6013
	LIFESTYLE = 6012
	MEDICAL = 6020
	MUSIC = 6011
	NAVIGATION = 6010
	NEWS = 6009
	NEWSSTAND = 6021
	PHOTO_AND_VIDEO = 6008
	PRODUCTIVITY = 6007
	REFERENCE = 6006
	SOCIAL_NETWORKING = 6005
	SPORTS = 6004
	TRAVEL = 6003
	UTILITIES = 6002
	WEATHER = 6001

class QueryTypeEnum:
	SEARCH = "Search"
	FETCH = "Fetch"


class GooglePlayStoreScraper(HTMLParser):
	'''
	A Google Play store Parser based on the inbuilt Python HTMLParser
	So that no other dependencies are required
	Use Class methods `fetch` or `search` to get Raw Data
	Provide this data to an OBJECT of this class via the `feed` method
	Thereafter the App List will be available in the `apps` object attribute
	'''
	PLAY_STORE_URL = "https://play.google.com/store/apps/collection/{collection}"
	PLAY_STORE_SEARCH_URL = "https://play.google.com/store/search?q={term}&c=apps"

	def __init__(self):
		self._current_app = None
		self.APP_STATE = None
		self.apps = []
		HTMLParser.__init__(self)

	@classmethod
	def fetch(cls, collection='topselling_free', start=0, num=0, **kargs):
		'''
		Fetch the data from a given or default URL
		:param str collection: The type of collection to query (top, paid, free, etc..)
		:param int start: For paging purposes, skip these many results in the query
		:param int num: The number of results to query for
		:param str url: If provided, overwrites the default Query URL
		:return: String result or None if URLException
		'''
		try:
			body = [('xhr', 1), ('ipf', 1)]
			if start: body.append(('start', start))
			if num: body.append(('num', num))

			# Either default URL, or one that is provided
			url = kargs.get('url', cls.PLAY_STORE_URL.format(collection=collection))

			body = urlencode(body)
			data = urlopen(url, data=body)
		except URLError as e:
			logging.getLogger().exception("Error in Play Store Query url:%s, collection:%s, start:%s, num:%s, kargs:%s, Reply:%s", \
				url, collection, start, num, kargs, e.read())
			return None

		return data.read()

	@classmethod
	def search(cls, term):
		'''
		Query a Search term website scrape
		:param str term: The search term to query for
		'''
		return cls.fetch(url=cls.PLAY_STORE_SEARCH_URL.format(term=quote(term)))

	
	# Below are overridden HTMLParser methods, to be used internally only
	# They are invoked when Data is provided to the Object via the 'feed' method

	def handle_starttag(self, tag, attrs):
		# convert list of tuples into easily referencable dict
		attrs_d = dict(attrs)

		# We have an App Card, <div class='card ... app' ...>
		if tag == 'div' and re.search('card.+app', attrs_d.get('class', '')): 
			#print attrs_d['data-docid'],
			# Overwrite any App we are currently on, it will remain in the Apps list
			self._current_app = dict(docid=attrs_d['data-docid'])

			# insert the current App, all changes to the above object will reflect in the list
			self.apps.append(self._current_app)

		# The App store link
		elif tag == 'a' and attrs_d.get('class') == 'card-click-target':
			self._current_app['link'] = attrs_d.get('href')

		# The App title
		elif tag == 'a' and attrs_d.get('class') == 'title' and attrs_d.has_key('title'):
			self._current_app['name'] = attrs_d.get('title')

		# The App Images
		elif tag == 'img' and attrs_d.get('class') == 'cover-image':
			self._current_app['image-large'] = attrs_d.get('data-cover-large')
			self._current_app['image-small'] = attrs_d.get('data-cover-small')

		# The App developer
		elif tag == 'a' and attrs_d.get('class') == 'subtitle':
			self._current_app['developer'] = attrs_d.get('title')

		elif tag == 'button' and attrs_d.get('class') == 'price buy':
			self._current_app['price'] = None # The next span will have the price, see handle_data

		# Description is a layer of nested p-tags within this div
		# So maintain state and keep adding description text while the flag is active
		elif tag == 'div' and attrs_d.get('class') == 'description':
			self.APP_STATE = 'in-description'
			self._current_app['description'] = ""
<commit_msg>Fixed play store urls<commit_after>from model.appinfo import AppInfo, AppInfoPlatforms

from urllib2 import urlopen, URLError
from urllib import urlencode, quote
from HTMLParser import HTMLParser
from xml.etree import ElementTree
import re

import json
import logging

class StoreTypeEnum:
	PLAY_STORE = "PlayStore"
	APP_STORE = "AppStore"

class CountryEnum:
	US = 'us'

class PlayStoreCollectionEnum:
	TOP_SELLING_FREE = "topselling_free"
	TOP_SELLING_PAID = "topselling_paid"
	TOP_GROSSING = "topgrossing"
	TOP_SELLING_NEW_FREE = "topselling_new_free"
	TOP_SELLING_NEW_PAID = "topselling_new_paid"

class AppStoreRSSTypeEnum:
	TOP_FREE = 'topfreeapplications'
	NEW_FREE = 'newfreeapplications'
	NEW_PAID = 'newpaidapplications'
	NEW_APPS = 'newapplications'
	TOP_GROSSING = 'topgrossingapplications'
	TOP_PAID = 'toppaidapplications'

class AppStoreRSSGenreEnum:
	ALL = None
	BOOKS = 6018
	BUSINESS = 6000
	CATALOGS = 6022
	EDUCATION = 6017
	ENTERTAINMENT = 6016
	FINANCE = 6015
	FOOD_AND_DRINK = 6023
	GAMES = 6014
	HEALTH_AND_FITNESS = 6013
	LIFESTYLE = 6012
	MEDICAL = 6020
	MUSIC = 6011
	NAVIGATION = 6010
	NEWS = 6009
	NEWSSTAND = 6021
	PHOTO_AND_VIDEO = 6008
	PRODUCTIVITY = 6007
	REFERENCE = 6006
	SOCIAL_NETWORKING = 6005
	SPORTS = 6004
	TRAVEL = 6003
	UTILITIES = 6002
	WEATHER = 6001

class QueryTypeEnum:
	SEARCH = "Search"
	FETCH = "Fetch"


class GooglePlayStoreScraper(HTMLParser):
	'''
	A Google Play store Parser based on the inbuilt Python HTMLParser
	So that no other dependencies are required
	Use Class methods `fetch` or `search` to get Raw Data
	Provide this data to an OBJECT of this class via the `feed` method
	Thereafter the App List will be available in the `apps` object attribute
	'''
	PLAY_STORE_URL = "https://play.google.com/store/apps/collection/{collection}"
	PLAY_STORE_SEARCH_URL = "https://play.google.com/store/search?q={term}&c=apps"

	def __init__(self):
		self._current_app = None
		self.APP_STATE = None
		self.apps = []
		HTMLParser.__init__(self)

	@classmethod
	def fetch(cls, collection='topselling_free', start=0, num=0, **kargs):
		'''
		Fetch the data from a given or default URL
		:param str collection: The type of collection to query (top, paid, free, etc..)
		:param int start: For paging purposes, skip these many results in the query
		:param int num: The number of results to query for
		:param str url: If provided, overwrites the default Query URL
		:return: String result or None if URLException
		'''
		try:
			body = [('xhr', 1), ('ipf', 1)]
			if start: body.append(('start', start))
			if num: body.append(('num', num))

			# Either default URL, or one that is provided
			url = kargs.get('url', cls.PLAY_STORE_URL.format(collection=collection))

			body = urlencode(body)
			data = urlopen(url, data=body)
		except URLError as e:
			logging.getLogger().exception("Error in Play Store Query url:%s, collection:%s, start:%s, num:%s, kargs:%s, Reply:%s", \
				url, collection, start, num, kargs, e.read())
			return None

		return data.read()

	@classmethod
	def search(cls, term):
		'''
		Query a Search term website scrape
		:param str term: The search term to query for
		'''
		return cls.fetch(url=cls.PLAY_STORE_SEARCH_URL.format(term=quote(term)))

	
	# Below are overridden HTMLParser methods, to be used internally only
	# They are invoked when Data is provided to the Object via the 'feed' method

	def handle_starttag(self, tag, attrs):
		# convert list of tuples into easily referencable dict
		attrs_d = dict(attrs)

		# We have an App Card, <div class='card ... app' ...>
		if tag == 'div' and re.search('card.+app', attrs_d.get('class', '')): 
			#print attrs_d['data-docid'],
			# Overwrite any App we are currently on, it will remain in the Apps list
			self._current_app = dict(docid=attrs_d['data-docid'])

			# insert the current App, all changes to the above object will reflect in the list
			self.apps.append(self._current_app)

		# The App store link
		elif tag == 'a' and attrs_d.get('class') == 'card-click-target':
			self._current_app['link'] = 'https://play.google.com' + attrs_d.get('href', '/no-link-found')

		# The App title
		elif tag == 'a' and attrs_d.get('class') == 'title' and attrs_d.has_key('title'):
			self._current_app['name'] = attrs_d.get('title')

		# The App Images
		elif tag == 'img' and attrs_d.get('class') == 'cover-image':
			self._current_app['image-large'] = attrs_d.get('data-cover-large')
			self._current_app['image-small'] = attrs_d.get('data-cover-small')

		# The App developer
		elif tag == 'a' and attrs_d.get('class') == 'subtitle':
			self._current_app['developer'] = attrs_d.get('title')

		elif tag == 'button' and attrs_d.get('class') == 'price buy':
			self._current_app['price'] = None # The next span will have the price, see handle_data

		# Description is a layer of nested p-tags within this div
		# So maintain state and keep adding description text while the flag is active
		elif tag == 'div' and attrs_d.get('class') == 'description':
			self.APP_STATE = 'in-description'
			self._current_app['description'] = ""

{'fraction_non_alphanumeric': 0.11274607277788824, 'fraction_numerical': 0.019288128852654605, 'mean_word_length': 4.396459227467811, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3093513', 'n_tokens_mistral': 3794, 'n_tokens_neox': 3447, 'n_words': 1041}