<commit_before>import time
import praw
from requests.exceptions import ConnectionError, HTTPError, Timeout, RequestException
import requests
import Config
from bs4 import BeautifulSoup
reddit = praw.Reddit(client_id=Config.cid,
                     client_secret=Config.secret,
                     password=Config.password,
                     user_agent=Config.agent,
                     username=Config.user)
subreddit = reddit.subreddit(Config.subreddit)
def flair(app_rating, num_installs, sub):
    inst = num_installs.split("+")
    if (inst[0] == "Couldn't"):
        return
    elif int(inst[0].replace(',', '')) <= 500:
        sub.mod.flair(text='New app', css_class=None)
    elif int(inst[0].replace(',', '')) >= 10000 and int(app_rating[0:1]) >= 4:
        sub.mod.flair(text= 'Popular app', css_class=None)
footer = """

*****

^^^[Source](https://github.com/a-ton/gpd-bot)
^^^|
^^^[Suggestions?](https://www.reddit.com/r/GPDBot/comments/7cyrfw)"""
file = open("postids.txt","a+")
file.close()
def logID(postid):
    f = open("postids.txt","a+")
    f.write(postid + "\n")
    f.close()

def crawl(s, u):
    print("Crawling...")
    page = requests.get(u).text
    store_page = BeautifulSoup(page, "html.parser")

    list_of_details = store_page.findAll(attrs={"class": "htlgb"})

	# get app name
    try:
        app_name = store_page.find("h1", class_="AHFaub").string
    except AttributeError:
        return "incorrect link"


	# get the number of downloads
    try:
        installs = list_of_details[2].string
    except AttributeError:
        installs = "  Couldn't get # of installs (probably a new app)"

	# get rating out of 5
    try:
        temp = store_page.find("div", class_="BHMmbe").string
        rating = temp + "/5"
    except TypeError:
        rating = "No ratings yet"

	# get developer name
    dev = store_page.find("a", class_="hrTbp R8zArc").string
    if dev in Config.blacklisted_devs:
        return "Sorry, deals from " + dev + " have been blacklisted.\n\nHere is the full list of blacklisted devleopers: https://www.reddit.com/r/googleplaydeals/wiki/blacklisted_devlopers"

	# get last update date
    updated = list_of_details[0].string

    # get size of app
    app_size = list_of_details[1].string

    # get current price
    temp = store_page.find("meta", itemprop="price")
    current_price = temp['content']
    if current_price == "0":
        current_price = "Free"

	# get full (normal) price
    try:
        full_price = store_page.find("span", class_="LV0gI").string
    except AttributeError:
        full_price = current_price + " (can't get price in USD)"

    # find IAPs
    iap_element = store_page.find("div", class_="rxic6")
    if iap_element == None:
        IAP = "No"
    else:
        IAP = "Yes"
    
    # get IAP info
    if (IAP == "Yes"):
        IAP_info = ", "<commit_msg>Fixed issue with apps that don't show a size<commit_after>import time
import praw
from requests.exceptions import ConnectionError, HTTPError, Timeout, RequestException
import requests
import Config
from bs4 import BeautifulSoup
reddit = praw.Reddit(client_id=Config.cid,
                     client_secret=Config.secret,
                     password=Config.password,
                     user_agent=Config.agent,
                     username=Config.user)
subreddit = reddit.subreddit(Config.subreddit)
def flair(app_rating, num_installs, sub):
    inst = num_installs.split("+")
    if (inst[0] == "Couldn't"):
        return
    elif int(inst[0].replace(',', '')) <= 500:
        sub.mod.flair(text='New app', css_class=None)
    elif int(inst[0].replace(',', '')) >= 10000 and int(app_rating[0:1]) >= 4:
        sub.mod.flair(text= 'Popular app', css_class=None)
footer = """

*****

^^^[Source](https://github.com/a-ton/gpd-bot)
^^^|
^^^[Suggestions?](https://www.reddit.com/r/GPDBot/comments/7cyrfw)"""
file = open("postids.txt","a+")
file.close()
def logID(postid):
    f = open("postids.txt","a+")
    f.write(postid + "\n")
    f.close()

def crawl(s, u):
    print("Crawling...")
    page = requests.get(u).text
    store_page = BeautifulSoup(page, "html.parser")

    list_of_details = store_page.findAll(attrs={"class": "htlgb"})

	# get app name
    try:
        app_name = store_page.find("h1", class_="AHFaub").string
    except AttributeError:
        return "incorrect link"


	# get the number of downloads
    installs = list_of_details[2].string
    try:
        inst = installs.split("+")
        int(inst[0].replace(',', ''))
    except ValueError:
        installs = list_of_details[1].string
        

	# get rating out of 5
    try:
        temp = store_page.find("div", class_="BHMmbe").string
        rating = temp + "/5"
    except TypeError:
        rating = "No ratings yet"

	# get developer name
    dev = store_page.find("a", class_="hrTbp R8zArc").string
    if dev in Config.blacklisted_devs:
        return "Sorry, deals from " + dev + " have been blacklisted.\n\nHere is the full list of blacklisted devleopers: https://www.reddit.com/r/googleplaydeals/wiki/blacklisted_devlopers"

	# get last update date
    updated = list_of_details[0].string

    # get size of app
    app_size = list_of_details[1].string
    try:
        inst = app_size.replace('M', '')
        int(inst)
    except ValueError:
        app_size = "Not given"
    print (app_size)

    # get current price
    temp = store_page.find("meta", itemprop="price")
    current_price = temp['content']
    if current_price == "0":
        current_price = "Free"

	# get full (normal) price
    try:
        full_price = store_page.find("span", class_="LV0gI").string
    except AttributeError:
        full_price = current_price + " (can't get price in USD)"

    # find IAPs
    iap_element = store_page.find("div", class_="rxic6")
    if iap_element == None:
        IAP = "No"
    else:
        IAP = "Yes"
    
    # get IAP info
    if (IAP == "Yes"):
        IAP_info = ", "
{'fraction_non_alphanumeric': 0.1420397172669135, 'fraction_numerical': 0.009087849209020532, 'mean_word_length': 2.871661237785016, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 6, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '67441', 'n_tokens_mistral': 2037, 'n_tokens_neox': 1981, 'n_words': 447}