<commit_before># -*- coding: utf-8 -*-


from lxml import etree, cssselect
import urllib2
import re
from StringIO import StringIO
import chardet


def urlload(url, format, param=None):
    res = urllib2.urlopen(url, param)
    document = ''.join(res)
    encoding = res.info().get('charset')
    if encoding is None:
        encoding = chardet.detect(document)['encoding']
    document = document.decode(encoding)
    return Parser.parse(format, document)


def load(s, format):
    return loads(''.join(s), format)


def loads(s, format):
    if isinstance(s, str):
        encoding = chardet.detect(s)['encoding']
        s = s.decode(encoding)
    return Parser.parse(format, s)


class Parser:

    @classmethod
    def parse(cls, format, document=None):
        xml = document
        if xml is not None and not isinstance(xml, etree.ElementBase):
            xml = etree.parse(StringIO(document), etree.HTMLParser())
        r = cls._parse(format, xml)
        (d, c) = r
        if c.lstrip() != '':  # if non-parsed data remain
            raise Xml2DataSyntaxError()
        return d

    @classmethod
    def _parse(cls, content, xml=None):
        """ parse a given content of the format.
            it returns a tuple (parsed data, remaining content).
        """
        c = content.lstrip()
        for f in (cls._parse_num, cls._parse_str) + ( 
                  lambda c_: cls._parse_dict(c_, xml),) + (
                  (lambda c_: cls._parse_selector(c_, xml),) if xml is not None else ()):
            try:
                r = f(c)
            except Xml2DataSyntaxError:
                continue
            return r
        raise Xml2DataSyntaxError()

    @classmethod
    def _parse_str(cls, content):
       c = content.lstrip()
       q = ('\'' if c[0] == '\'' else
            '\"' if c[0] == '\"' else None)
       if q is None:
           raise Xml2DataSyntaxError()
       s = 1
       while True:
           i = c.find(q, s)
           if i == -1:
               raise Xml2DataSyntaxError()
           is_valid_quote = True
           for j in range(i - 1, 0, -2):
               if c[j] != '\\':
                   break
               if c[j] == '\\' and c[j - 1: j + 1] != '\\\\':
                   is_valid_quote = False
                   c = c[0:j] + c[j + 1:]
                   s = i
                   break
           if is_valid_quote:
               return (c[1:i], c[i + 1:])

    @classmethod
    def _parse_num(cls, content):
        c = content.lstrip()
        m = re.match(r'[\-\+]?\d+', c)
        if m is None:
            raise Xml2DataSyntaxError()
        return (int(m.group()), c[m.end():])

    @classmethod
    def _parse_dict(cls, content, *args):
        c = content.lstrip()

        if c[0] != '{':
            raise Xml2DataSyntaxError()

        d = {}
        c = c[1:].lstrip()
        while c[0] != '}':
            r = cls._parse(c, *args)
            (k, c) = r

            c = c.lstrip()
            if c[0] != ':':
                raise Xml2DataSyntaxError()

            c = c[1:].lstrip()
            r = cls._parse(c, *args)
            (v, c) = r

            c = c.lstrip()
            if not (c[0] == ',' or c[0] == '}'):
                raise Xml2DataSyntaxError()

            try:
                d[k] = v
            except TypeError:
                pass
            except:
                raise Xml2DataSyntaxError()

            if c[0] == ',':
                c = c[1:].lstrip()

        return (d, c[1:])

    # not used
    @classmethod
    def _parse_regular_list(cls, content, *args):
        c = content.lstrip()
        if c[0] != '[':
            raise Xml2DataSyntaxError()

        d = []
        c = c[1:].lstrip()
        while c[0] != ']':
            r = cls._parse(c, *args)
            (v, c) = r

            c = c.lstrip()
            if not (c[0] == ',' or c[0] == ']'):
                raise Xml2DataSyntaxError()

            d.append(v)

            if c[0] == ',':
                c = c[1:].lstrip()

        return (d, c[1:])

    @classmethod
    def _parse_selector(cls, content, xml):
<commit_msg>a few modification just for readability<commit_after># -*- coding: utf-8 -*-


from lxml import etree, cssselect
import urllib2
import re
from StringIO import StringIO
import chardet


def urlload(url, format, param=None):
    res = urllib2.urlopen(url, param)
    document = ''.join(res)
    encoding = res.info().get('charset') or chardet.detect(document)['encoding']
    document = document.decode(encoding)
    return Parser.parse(format, document)


def load(s, format):
    return loads(''.join(s), format)


def loads(s, format):
    if isinstance(s, str):
        encoding = chardet.detect(s)['encoding']
        s = s.decode(encoding)
    return Parser.parse(format, s)


class Parser:

    @classmethod
    def parse(cls, format, document=None):
        xml = document
        if xml is not None and not isinstance(xml, etree.ElementBase):
            xml = etree.parse(StringIO(document), etree.HTMLParser())
        (d, c) = cls._parse(format, xml)
        if c.lstrip() != '':  # if non-parsed data remain
            raise Xml2DataSyntaxError()
        return d

    @classmethod
    def _parse(cls, content, xml=None):
        """ parse a given content of the format.
            it returns a tuple (parsed data, remaining content).
        """
        c = content.lstrip()
        for f in (cls._parse_num, cls._parse_str) + ( 
                  lambda c_: cls._parse_dict(c_, xml),) + (
                  (lambda c_: cls._parse_selector(c_, xml),) if xml is not None else ()):
            try:
                return f(c)
            except Xml2DataSyntaxError:
                continue
        raise Xml2DataSyntaxError()

    @classmethod
    def _parse_str(cls, content):
       c = content.lstrip()
       q = ('\'' if c[0] == '\'' else
            '\"' if c[0] == '\"' else None)
       if q is None:
           raise Xml2DataSyntaxError()
       s = 1
       while True:
           i = c.find(q, s)
           if i == -1:
               raise Xml2DataSyntaxError()
           is_valid_quote = True
           for j in range(i - 1, 0, -2):
               if c[j] != '\\':
                   break
               if c[j] == '\\' and c[j - 1: j + 1] != '\\\\':
                   is_valid_quote = False
                   c = c[0:j] + c[j + 1:]
                   s = i
                   break
           if is_valid_quote:
               return (c[1:i], c[i + 1:])

    @classmethod
    def _parse_num(cls, content):
        c = content.lstrip()
        m = re.match(r'[\-\+]?\d+', c)
        if m is None:
            raise Xml2DataSyntaxError()
        return (int(m.group()), c[m.end():])

    @classmethod
    def _parse_dict(cls, content, *args):
        c = content.lstrip()

        if c[0] != '{':
            raise Xml2DataSyntaxError()

        d = {}
        c = c[1:].lstrip()
        while c[0] != '}':
            (k, c) = cls._parse(c, *args)

            c = c.lstrip()
            if c[0] != ':':
                raise Xml2DataSyntaxError()

            c = c[1:].lstrip()
            (v, c) = cls._parse(c, *args)

            c = c.lstrip()
            if not (c[0] == ',' or c[0] == '}'):
                raise Xml2DataSyntaxError()

            try:
                d[k] = v
            except TypeError:
                pass
            except:
                raise Xml2DataSyntaxError()

            if c[0] == ',':
                c = c[1:].lstrip()

        return (d, c[1:])

    # not used
    @classmethod
    def _parse_regular_list(cls, content, *args):
        c = content.lstrip()
        if c[0] != '[':
            raise Xml2DataSyntaxError()

        d = []
        c = c[1:].lstrip()
        while c[0] != ']':
            (v, c) = cls._parse(c, *args)

            c = c.lstrip()
            if not (c[0] == ',' or c[0] == ']'):
                raise Xml2DataSyntaxError()

            d.append(v)

            if c[0] == ',':
                c = c[1:].lstrip()

        return (d, c[1:])

    @classmethod
    def _parse_selector(cls, content, xml):

{'fraction_non_alphanumeric': 0.14292748984365383, 'fraction_numerical': 0.01132586482826542, 'mean_word_length': 1.4889705882352942, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6137777', 'n_tokens_mistral': 2684, 'n_tokens_neox': 2571, 'n_words': 667}