<commit_before>import requests
import json
from bs4 import BeautifulSoup
import random
import time
import traceback

class PlayReviews():
	
	def get_reviews(self, app_id):
		sort_by_time = '0'
		sort_by_helpfulness = '2'
		sort_by_time = '1'
		# app_id = "com.buyhatke.assistant"
		reviews = []
		page_num = 0;
		is_last_page = False

		while is_last_page is False:
			(reviews_html_string, is_last_page) = self.__fetch_reviews(app_id, page_num, sort_by_helpfulness)
			if len(reviews_html_string) > 0:
				reviews.extend(self.__parse_reviews(reviews_html_string))
			page_num = page_num + 1

		return reviews

	def __fetch_reviews(self, app_id, page_num, sort_by):
		url = "https://play.google.com/store/getreviews"
		querystring = {"authuser":"0"}
		payload = {'reviewType':'0', 'pageNum':page_num, 'id':app_id, 'xhr':'1', 'reviewSortOrder':sort_by}
		response = requests.request("POST", url, data=payload, params=querystring)
		
		if(len(response.content)>4):
			response = response.content[4:]
			# print(response)
			try:
				response = json.loads(response)
				reviews_html = response[0][2]
				if response[0][1] == 1:
					is_last_page = False
				else:
					is_last_page = True
				return (reviews_html, is_last_page)
			
			except ValueError as e:
				traceback.print_exc()
				print("google blocked you, maybe sleep")
				return('',True)
		else:
			return('', True)

	def __parse_reviews(self, reviews_html_string):
		soup = BeautifulSoup(reviews_html_string, 'html.parser')
		reviews = []
		review_body_list = soup.findAll("div",{"class":"review-body"})
		review_author_list = soup.findAll("span",{"class":"author-name"})
		review_date_list = soup.findAll("span",{"class":"review-date"})
		review_title_list = soup.findAll("span",{"class":"review-title"})
		review_rating_list = soup.findAll("div",{"class":"tiny-star"})
		review_user_id_list = soup.findAll()
		
		for i in range(len(review_body_list)):
			review = {"title":review_title_list[i].text, "body":review_body_list[i].text, 
			"date":review_date_list[i].text, "author":review_author_list[i].text, "rating":review_rating_list[i]['aria-label']}
			reviews.append(review)
		return reviews<commit_msg>Converted tabs to spaces on special demand<commit_after>import requests
import json
from bs4 import BeautifulSoup
import random
import time
import traceback

class PlayReviews():
    
    def get_reviews(self, app_id):
        sort_by_time = '0'
        sort_by_helpfulness = '2'
        sort_by_time = '1'
        # app_id = "com.buyhatke.assistant"
        reviews = []
        page_num = 0;
        is_last_page = False

        while is_last_page is False:
            (reviews_html_string, is_last_page) = self.__fetch_reviews(app_id, page_num, sort_by_helpfulness)
            if len(reviews_html_string) > 0:
                reviews.extend(self.__parse_reviews(reviews_html_string))
            page_num = page_num + 1

        return reviews

    def __fetch_reviews(self, app_id, page_num, sort_by):
        url = "https://play.google.com/store/getreviews"
        querystring = {"authuser":"0"}
        payload = {'reviewType':'0', 'pageNum':page_num, 'id':app_id, 'xhr':'1', 'reviewSortOrder':sort_by}
        response = requests.request("POST", url, data=payload, params=querystring)
        
        if(len(response.content)>4):
            response = response.content[4:]
            # print(response)
            try:
                response = json.loads(response)
                reviews_html = response[0][2]
                if response[0][1] == 1:
                    is_last_page = False
                else:
                    is_last_page = True
                return (reviews_html, is_last_page)
            
            except ValueError as e:
                traceback.print_exc()
                print("google blocked you, maybe sleep")
                return('',True)
        else:
            return('', True)

    def __parse_reviews(self, reviews_html_string):
        soup = BeautifulSoup(reviews_html_string, 'html.parser')
        reviews = []
        review_body_list = soup.findAll("div",{"class":"review-body"})
        review_author_list = soup.findAll("span",{"class":"author-name"})
        review_date_list = soup.findAll("span",{"class":"review-date"})
        review_title_list = soup.findAll("span",{"class":"review-title"})
        review_rating_list = soup.findAll("div",{"class":"tiny-star"})
        review_user_id_list = soup.findAll()
        
        for i in range(len(review_body_list)):
            review = {"title":review_title_list[i].text, "body":review_body_list[i].text, 
            "date":review_date_list[i].text, "author":review_author_list[i].text, "rating":review_rating_list[i]['aria-label']}
            reviews.append(review)
        return reviews
{'fraction_non_alphanumeric': 0.13113384840258926, 'fraction_numerical': 0.007099603257465024, 'mean_word_length': 3.2277140335392764, 'pattern_counts': {'":': 22, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5124673', 'n_tokens_mistral': 1682, 'n_tokens_neox': 1541, 'n_words': 277}