<commit_before>#!/usr/bin/python
import sys
import os
import logging
from optparse import OptionParser
from io import StringIO
import json
import locale

from lxml import etree
from lxml.etree import ParserError
from lxml.etree import XMLSyntaxError
import requests
from requests.exceptions import ConnectionError
from requests.exceptions import HTTPError
from requests.exceptions import Timeout
from requests.exceptions import ReadTimeout

import xpathExpressions


class PlayListingScraper(object):
    log = logging.getLogger("play_listing_scraper")
    # The logger's level must be set to the "lowest" level.
    log.setLevel(logging.DEBUG)
    locale.setlocale( locale.LC_ALL, 'en_US.UTF-8' )

    def parse_tree(self, tree, package_name, version_code, out_dir):
        title = self.get_property(tree, xpathExpressions.TITLE)
        description = self.get_property(tree, xpathExpressions.DESCRIPTION)
        category = self.get_property(tree, xpathExpressions.CATEGORY)
        price = self.get_property(tree, xpathExpressions.PRICE)
        date_published = self.get_property(tree, xpathExpressions.DATE_PUBLISHED)
        os_version = self.get_property(tree, xpathExpressions.OPERATING_SYSTEM)
        rating_count_text = self.get_property(tree, xpathExpressions.RATING_COUNT)
        rating_count = locale.atoi(rating_count_text)
        rating_text = self.get_property(tree, xpathExpressions.RATING)
        rating = float(rating_text.split()[1])
        content_rating = self.get_property(tree, xpathExpressions.CONTENT_RATING)
        creator = self.get_property(tree, xpathExpressions.CREATOR)
        creator_url = self.get_property(tree, xpathExpressions.CREATOR_URL)
        creator_address = self.get_property(tree, xpathExpressions.CREATOR_ADDRESS)
        install_size = self.get_property(tree, xpathExpressions.INSTALL_SIZE)
        download_count_text = self.get_property(tree,
                                                xpathExpressions.DOWNLOAD_COUNT_TEXT)
        download_count = locale.atoi(download_count_text.split("-")[0].strip())
        privacy_url = self.get_property(tree, xpathExpressions.PRIVACY_URL)
        whats_new = self.get_property(tree, xpathExpressions.WHATS_NEW)

        app = json.dumps({"n": package_name,
                          "verc": version_code,
                          "t": title,
                          "desc": description,
                          "cat": category,
                          "pri": price,
                          "dtp": date_published,
                          "os": os_version,
                          "rct" : rating_count,
                          "rate" : rating,
                          "crat" : content_rating,
                          "crt" : creator,
                          "cadd": creator_address,
                          "curl" : creator_url,
                          "sz" : install_size,
                          "dct" : download_count,
                          "dtxt" : download_count_text,
                          "purl": privacy_url,
                          "new" : whats_new
                          }, sort_keys=True, indent=4, separators=(',', ': '))
        out_file = os.path.join(out_dir, package_name + '-' + version_code +
                                '.listing.json')
        with open(out_file, 'w') as f:
            f.write(app)
        self.log.info("Listing details info has been written at %s", out_file)

    @staticmethod
    def get_property(tree, xpath_expressions):
        val = None
        for exp in xpath_expressions:
            val = tree.xpath(exp)
            if val:
                break
        return ' '.join(val).strip()

    def scrape_content(self, html_content, package_name, version_code,
                       out_dir=os.getcwd()):
        try:
            parser = etree.HTMLParser()
            tree = etree.parse(html_content, parser)
            if tree.getroot() is None:
                self.log.error("Invalid html content")
            else:
                self.parse_tree(tree, package_name, version_code, out_dir)
        except (ParserError, XMLSyntaxError) as e:
            self.log.error("Error in parsing html page for %s message: %s",
                           package_name + "-" + version_code, e)

    def scrape_remote_page(self, package_name, version_code,
                           out_dir=os.getcwd()):
        try:
            play_url = "https://play.google.com/store/apps/details?id=" + \
                       package_name
            r = requests.get(play_url)
            if r.status_code == 200:
                html_content = StringIO(requests.get(play_url).text)
                self.scrape_content(html_content, package_name, version_code,
                                    out_dir)
            else:
                self.log.error("Failed to request the html page for %s " +
                               "http request status code: %i", package_name,
                               r.status_code)
        except (ConnectionError, HTTPError, Timeout, ReadTimeout) as e:
            self.log.error("Network error while requesting the html page for" +
                           " %s message: %s",
                           package_name + "-" + version_code, e)

    def cli(self, argv):
        out_dir = os.getcwd()
        logging_file = None
        logging_level = logging.ERROR
        # Create formatter
        formatter = logging.Formatter(
            '%(asctime)s - %(levelname)s - %(message)s')
        # Create console logger and set its formatter and level
        logging_console = logging.StreamHandler(sys.stdout)
        logging_console.setFormatter(formatter)
        logging_console.setLevel(logging.DEBUG)
        # Add the console logger
        self.log.addHandler(logging_console)
        parser = OptionParser(
            usage="%prog [options] " +
                  "packageName-versionCode | google_play_html_page_file",
            version="%prog 0.4",
            description="A tool for Scraping an app's listing details data " +
                        "from the Google Play Store store " +
                        "and saving it in a file in JSON format.")<commit_msg>fix version number<commit_after>#!/usr/bin/python
import sys
import os
import logging
from optparse import OptionParser
from io import StringIO
import json
import locale

from lxml import etree
from lxml.etree import ParserError
from lxml.etree import XMLSyntaxError
import requests
from requests.exceptions import ConnectionError
from requests.exceptions import HTTPError
from requests.exceptions import Timeout
from requests.exceptions import ReadTimeout
from __init__ import __version__

import xpathExpressions


class PlayListingScraper(object):
    log = logging.getLogger("play_listing_scraper")
    # The logger's level must be set to the "lowest" level.
    log.setLevel(logging.DEBUG)
    locale.setlocale( locale.LC_ALL, 'en_US.UTF-8' )

    def parse_tree(self, tree, package_name, version_code, out_dir):
        title = self.get_property(tree, xpathExpressions.TITLE)
        description = self.get_property(tree, xpathExpressions.DESCRIPTION)
        category = self.get_property(tree, xpathExpressions.CATEGORY)
        price = self.get_property(tree, xpathExpressions.PRICE)
        date_published = self.get_property(tree, xpathExpressions.DATE_PUBLISHED)
        os_version = self.get_property(tree, xpathExpressions.OPERATING_SYSTEM)
        rating_count_text = self.get_property(tree, xpathExpressions.RATING_COUNT)
        rating_count = locale.atoi(rating_count_text)
        rating_text = self.get_property(tree, xpathExpressions.RATING)
        rating = float(rating_text.split()[1])
        content_rating = self.get_property(tree, xpathExpressions.CONTENT_RATING)
        creator = self.get_property(tree, xpathExpressions.CREATOR)
        creator_url = self.get_property(tree, xpathExpressions.CREATOR_URL)
        creator_address = self.get_property(tree, xpathExpressions.CREATOR_ADDRESS)
        install_size = self.get_property(tree, xpathExpressions.INSTALL_SIZE)
        download_count_text = self.get_property(tree,
                                                xpathExpressions.DOWNLOAD_COUNT_TEXT)
        download_count = locale.atoi(download_count_text.split("-")[0].strip())
        privacy_url = self.get_property(tree, xpathExpressions.PRIVACY_URL)
        whats_new = self.get_property(tree, xpathExpressions.WHATS_NEW)

        app = json.dumps({"n": package_name,
                          "verc": version_code,
                          "t": title,
                          "desc": description,
                          "cat": category,
                          "pri": price,
                          "dtp": date_published,
                          "os": os_version,
                          "rct" : rating_count,
                          "rate" : rating,
                          "crat" : content_rating,
                          "crt" : creator,
                          "cadd": creator_address,
                          "curl" : creator_url,
                          "sz" : install_size,
                          "dct" : download_count,
                          "dtxt" : download_count_text,
                          "purl": privacy_url,
                          "new" : whats_new
                          }, sort_keys=True, indent=4, separators=(',', ': '))
        out_file = os.path.join(out_dir, package_name + '-' + version_code +
                                '.listing.json')
        with open(out_file, 'w') as f:
            f.write(app)
        self.log.info("Listing details info has been written at %s", out_file)

    @staticmethod
    def get_property(tree, xpath_expressions):
        val = None
        for exp in xpath_expressions:
            val = tree.xpath(exp)
            if val:
                break
        return ' '.join(val).strip()

    def scrape_content(self, html_content, package_name, version_code,
                       out_dir=os.getcwd()):
        try:
            parser = etree.HTMLParser()
            tree = etree.parse(html_content, parser)
            if tree.getroot() is None:
                self.log.error("Invalid html content")
            else:
                self.parse_tree(tree, package_name, version_code, out_dir)
        except (ParserError, XMLSyntaxError) as e:
            self.log.error("Error in parsing html page for %s message: %s",
                           package_name + "-" + version_code, e)

    def scrape_remote_page(self, package_name, version_code,
                           out_dir=os.getcwd()):
        try:
            play_url = "https://play.google.com/store/apps/details?id=" + \
                       package_name
            r = requests.get(play_url)
            if r.status_code == 200:
                html_content = StringIO(requests.get(play_url).text)
                self.scrape_content(html_content, package_name, version_code,
                                    out_dir)
            else:
                self.log.error("Failed to request the html page for %s " +
                               "http request status code: %i", package_name,
                               r.status_code)
        except (ConnectionError, HTTPError, Timeout, ReadTimeout) as e:
            self.log.error("Network error while requesting the html page for" +
                           " %s message: %s",
                           package_name + "-" + version_code, e)

    def cli(self, argv):
        out_dir = os.getcwd()
        logging_file = None
        logging_level = logging.ERROR
        # Create formatter
        formatter = logging.Formatter(
            '%(asctime)s - %(levelname)s - %(message)s')
        # Create console logger and set its formatter and level
        logging_console = logging.StreamHandler(sys.stdout)
        logging_console.setFormatter(formatter)
        logging_console.setLevel(logging.DEBUG)
        # Add the console logger
        self.log.addHandler(logging_console)
        parser = OptionParser(
            usage="%prog [options] " +
                  "packageName-versionCode | google_play_html_page_file",
            version="%prog " + __version__,
            description="A tool for Scraping an app's listing details data " +
                        "from the Google Play Store store " +
                        "and saving it in a file in JSON format.")
{'fraction_non_alphanumeric': 0.08747291549634861, 'fraction_numerical': 0.0012840060990289704, 'mean_word_length': 1.90625, 'pattern_counts': {'":': 20, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5710954', 'n_tokens_mistral': 3460, 'n_tokens_neox': 3376, 'n_words': 749}