<commit_before>import web
import json
import os
import steam
import re
from HTMLParser import HTMLParser
from optf2.backend import database
from optf2.backend import config

# Subapplication and URLs defined after classes
# TODO: Well defined error objects for when this is
# more fleshed out

jsonMimeType = "application/json"

class jsonHTTPError(web.HTTPError):
    """" Replacement for web.* stuff,
    they set content-types to text/html by default.
    Not sure if two headers of those are allowed in the spec. """
    def __init__(self, status, message = None):
        headers = {"Content-Type": jsonMimeType}
        web.HTTPError.__init__(self, status, headers, message or "{}")

class jsonBadRequest(jsonHTTPError):
    """ Replacement for web.BadRequest """
    def __init__(self, message = None):
        status = "400 Bad Request"
        jsonHTTPError.__init__(self, status, message)

class jsonNotFound(jsonHTTPError):
    """ Replacement for web.NotFound """
    def __init__(self, message = None):
        status = "404 Not Found"
        jsonHTTPError.__init__(self, status, message)

class jsonInternalError(jsonHTTPError):
    """ Replacement for web._InternalError
    no stack calls needed here really """
    def __init__(self, message = None):
        status = "500 Internal Server Error"
        jsonHTTPError.__init__(self, status, message)

class search_page_parser(HTMLParser):
    def handle_starttag(self, tag, attrs):
        attr = dict(attrs)
        link = attr.get("href")
        aclass = attr.get("class")

        self._tagstack.append((tag, aclass))

        if aclass == "linkTitle" and link and link.startswith(self._community_url):
            # id64 is a misnomer, but is there for compatibility reasons
            self._obj["id64"] = os.path.basename(link)

            if link.startswith(self._community_url + "profiles"): self._obj["id_type"] = "id64"
            else: self._obj["id_type"] = "id"

        img = attr.get("src")
        if tag == "img" and img and img.find("avatars") != -1:
            self._obj["avatarurl"] = img.replace("_medium", "")

    def handle_data(self, data):
        try:
            lasttag, aclass = self._tagstack[-1]
            if aclass and aclass == "linkTitle":
                self._obj["persona"] = data.strip()
        except IndexError:
            pass

    def handle_endtag(self, tag):
        stag, aclass = self._tagstack.pop()
        if aclass and aclass.find("resultItem") != -1:
            if self._str != self._obj["id64"].lower() and self._str != str(self._prof.get("id64")):
                self._results.append(self._obj)
            self._obj = {}

    def get_results(self):
        return self._results

    def __init__(self, user, prof = None):
        self._obj = {}
        self._tagstack = []
        self._community_url = "http://steamcommunity.com/"
        self._results = []
        self._prof = prof or {}
        self._str = user.lower()
        search_url = self._community_url + "actions/Search?T=Account&K={0}".format(web.urlquote(user))

        HTMLParser.__init__(self)

        req = steam.json_request(search_url, timeout = config.ini.getint("steam", "connect-timeout"),
                                 data_timeout = config.ini.getint("steam", "download-timeout"))

        self.feed(req._download())

class group_member_page_parser(HTMLParser):
    _member_count_exp = re.compile("[\d,]+ - [\d,]+ of ([\d,]+) Members")

    def handle_starttag(self, tag, attrs):
        attr = dict(attrs)
        aclass = attr.get("class", '').strip().split(' ')

        self._tagstack.append((tag, aclass))

        if tag == "img":
            for tag, tclass in reversed(self._tagstack):
                src = attr.get("src", '')
                if "playerAvatar" in tclass:
                    self._obj["avatar"] = src
                    # May not be reliable, but there's no
                    # other source besides friendSmallText,
                    # which seems even less reliable
                    if len(tclass) > 1:
                        self._obj["status"] = tclass[1]
                    break
                elif "grouppage_logo" in tclass:
                    self._logo = src
                    break
        elif tag == "a" and "linkFriend" in aclass:
            self._obj["profile"] = attr.get("href")

    def handle_data(self, data):
        mcount = self._member_count_exp.search(data)

        try:
            if mcount: self._member_count = int(mcount.group(1).replace(',', ''))
        except TypeError:
            pass

        try:
            stag, sclass = self._tagstack[-1]
        except IndexError:
            return

        data = data.strip()<commit_msg>Don't set another logo if one is already set when traversing tags<commit_after>import web
import json
import os
import steam
import re
from HTMLParser import HTMLParser
from optf2.backend import database
from optf2.backend import config

# Subapplication and URLs defined after classes
# TODO: Well defined error objects for when this is
# more fleshed out

jsonMimeType = "application/json"

class jsonHTTPError(web.HTTPError):
    """" Replacement for web.* stuff,
    they set content-types to text/html by default.
    Not sure if two headers of those are allowed in the spec. """
    def __init__(self, status, message = None):
        headers = {"Content-Type": jsonMimeType}
        web.HTTPError.__init__(self, status, headers, message or "{}")

class jsonBadRequest(jsonHTTPError):
    """ Replacement for web.BadRequest """
    def __init__(self, message = None):
        status = "400 Bad Request"
        jsonHTTPError.__init__(self, status, message)

class jsonNotFound(jsonHTTPError):
    """ Replacement for web.NotFound """
    def __init__(self, message = None):
        status = "404 Not Found"
        jsonHTTPError.__init__(self, status, message)

class jsonInternalError(jsonHTTPError):
    """ Replacement for web._InternalError
    no stack calls needed here really """
    def __init__(self, message = None):
        status = "500 Internal Server Error"
        jsonHTTPError.__init__(self, status, message)

class search_page_parser(HTMLParser):
    def handle_starttag(self, tag, attrs):
        attr = dict(attrs)
        link = attr.get("href")
        aclass = attr.get("class")

        self._tagstack.append((tag, aclass))

        if aclass == "linkTitle" and link and link.startswith(self._community_url):
            # id64 is a misnomer, but is there for compatibility reasons
            self._obj["id64"] = os.path.basename(link)

            if link.startswith(self._community_url + "profiles"): self._obj["id_type"] = "id64"
            else: self._obj["id_type"] = "id"

        img = attr.get("src")
        if tag == "img" and img and img.find("avatars") != -1:
            self._obj["avatarurl"] = img.replace("_medium", "")

    def handle_data(self, data):
        try:
            lasttag, aclass = self._tagstack[-1]
            if aclass and aclass == "linkTitle":
                self._obj["persona"] = data.strip()
        except IndexError:
            pass

    def handle_endtag(self, tag):
        stag, aclass = self._tagstack.pop()
        if aclass and aclass.find("resultItem") != -1:
            if self._str != self._obj["id64"].lower() and self._str != str(self._prof.get("id64")):
                self._results.append(self._obj)
            self._obj = {}

    def get_results(self):
        return self._results

    def __init__(self, user, prof = None):
        self._obj = {}
        self._tagstack = []
        self._community_url = "http://steamcommunity.com/"
        self._results = []
        self._prof = prof or {}
        self._str = user.lower()
        search_url = self._community_url + "actions/Search?T=Account&K={0}".format(web.urlquote(user))

        HTMLParser.__init__(self)

        req = steam.json_request(search_url, timeout = config.ini.getint("steam", "connect-timeout"),
                                 data_timeout = config.ini.getint("steam", "download-timeout"))

        self.feed(req._download())

class group_member_page_parser(HTMLParser):
    _member_count_exp = re.compile("[\d,]+ - [\d,]+ of ([\d,]+) Members")

    def handle_starttag(self, tag, attrs):
        attr = dict(attrs)
        aclass = attr.get("class", '').strip().split(' ')

        self._tagstack.append((tag, aclass))

        if tag == "img":
            for tag, tclass in reversed(self._tagstack):
                src = attr.get("src", '')
                if "playerAvatar" in tclass:
                    self._obj["avatar"] = src
                    # May not be reliable, but there's no
                    # other source besides friendSmallText,
                    # which seems even less reliable
                    if len(tclass) > 1:
                        self._obj["status"] = tclass[1]
                    break
                elif not self._logo and "grouppage_logo" in tclass:
                    self._logo = src
                    break
        elif tag == "a" and "linkFriend" in aclass:
            self._obj["profile"] = attr.get("href")

    def handle_data(self, data):
        mcount = self._member_count_exp.search(data)

        try:
            if mcount: self._member_count = int(mcount.group(1).replace(',', ''))
        except TypeError:
            pass

        try:
            stag, sclass = self._tagstack[-1]
        except IndexError:
            return

        data = data.strip()
{'fraction_non_alphanumeric': 0.12067696835908756, 'fraction_numerical': 0.006096920004204772, 'mean_word_length': 2.2493169398907105, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4770989', 'n_tokens_mistral': 2778, 'n_tokens_neox': 2716, 'n_words': 715}