<commit_before>import re
import textwrap

import BeautifulSoup
from lxml.html import parse

from mtglib.gatherer_request import CardRequest

__all__ = ['CardExtractor', 'SingleCardExtractor', 'Card']

class CardExtractor(object):
    """Extracts card information from Gatherer HTML."""

    def __init__(self, html):
        self.html = html

    def _group(self, lst, n):
        newlist = []
        for i in range(0, len(lst), n):
            val = lst[i:i+n]
            if len(val) == n:
                newlist.append(tuple(val))
        return newlist

    def _flatten(self, element):
        """Recursively enter and extract text from all child
        elements."""
        result = [ (element.text or '') ]
        if element.attrib.get('alt'):
            result.append(Symbol(element.attrib.get('alt')).textbox)
        for sel in element:
            result.append(self._flatten(sel))
            result.append(sel.tail or '')
        return ''.join(result)

    def extract_many(self):
        doc = parse(self.html).getroot()
        cards = []
        for item in doc.cssselect('tr.cardItem'):
            for c in item.cssselect('div.cardInfo'):
                card = Card()
                card.name = c.cssselect('span.cardTitle')[0].text_content().strip()
                for img in c.cssselect('span.manaCost img'):
                    setattr(card, 'mana_cost', Symbol(img.attrib['alt']).short)
                regex = '\([^/]+/[^)]+\)'
                typeline = c.cssselect('span.typeLine')[0].text_content()
                m = re.search(regex, typeline)
                if m:
                    card.pow_tgh = m.group(0)
                    card.types = re.sub(regex, '', typeline).strip()
                else:
                    card.types = typeline.strip()
                card.types = card.types.replace(u'\xe2\x80\x94', u'\u2014')
                card.card_text = self._flatten(c.cssselect('div.rulesText')[0]).strip()

            set_rarity = ', '.join([img.attrib['alt'] for img in
                                    item.cssselect('td.rightCol img')])
            setattr(card, 'set_rarity', set_rarity)
            cards.append(card)
        return cards

    def extract(self, get_card_urls=False):
        if not self.html:
            return False

        doc = parse(self.html).getroot()
        cards = []
        for component in doc.cssselect('td.cardComponentContainer'):
            if not component.getchildren():
                continue # do not parse empty components
            labels = component.cssselect('div.label')
            values = component.cssselect('div.value')
            pairs = zip(labels, values)
            card = Card()
            for (label, value) in pairs:
                l = label.text_content().strip().replace(' ', '_') \
                    .replace(':', '').lower().replace('#', 'number')

                if l == 'card_text':
                    v = ' ; '.join(map(self._flatten,
                                       value.cssselect('div.cardtextbox')))
                else:
                    v = u''
                    if l == 'mana_cost':
                        for img in value.cssselect('img'):
                            v += Symbol(img.attrib['alt']).short
                    v += value.text_content().strip()
                setattr(card, l, v.replace(u'\xe2\x80\x94', u'\u2014'))
            cards.append(card)
        return cards



        # soup = BeautifulSoup.BeautifulSoup(self.html)
        # if not soup.table:
        #     return []
        # for tag in soup.findAll('br'):
        #     tag.replaceWith('||')

        # td_tags = soup.table.findAll('td')

        # # Get rulings hrefs here.
        # if get_card_urls:
        #     a_tags = soup.table.findAll('a')
        #     card_urls = [tag['href'] for tag in a_tags]

        # content_lists = [tag.contents for tag in td_tags]
        # unified_content = []
        # cards = []
        # for lst in content_lists:
        #     unified_content.append(''.join([item.string or u'' for item in lst]))

        # unified_content = [item for item in unified_content if item != u'\n||\n']
        # unified_content = self._group(unified_content, 2)

        # blocks  = []
        # block = []
        # for u in unified_content:
        #     block.append(u)
        #     if 'Set/Rarity' in u[0]:
        #         blocks.append(block)
        #         block = []

        # for block in blocks:
        #     card = Card.from_block(block)
        #     if get_card_urls:
        #         card.url = card_urls.pop(0)
        #     cards.append(card)
        # return cards

class Symbol(object):

    def __init__(self, text):
        self.text = text
        self.specials = {'Untap': 'Q',
                         'Blue': 'U',
                         'Snow': 'S}i',
                         'Variable Colorless': 'X'}

    @property
    def short(self):
        if self.text in self.specials.keys():
            return self.specials[self.text]
        elif ' or ' in self.text:
            return '({0})'.format('/'.join(
                    Symbol(l).short for l in self.text.split(' or ')))
        elif 'Phyrexian' in self.text:
            return self.phyrexian
        elif self.text.isdigit():
            return self.text
        return self.text[:1]

    @property
    def phyrexian(self):
        return '({0}/P)'.format(
            Symbol(self.text.replace('Phyrexian ', '')).short)

    @property
    def textbox(self):
        base = '{{{0}}}'.format(self.short)
        if '/' in base: return base.lower()
        else: return base


class SingleCardExtractor(object):

    def __init__(self, html):
        self.html = html

    def _parse_html(self):
        if not self.html:
            return False

    def extract_flavor(self):<commit_msg>More symbol refactoring.<commit_after>import re
import textwrap

import BeautifulSoup
from lxml.html import parse

from mtglib.gatherer_request import CardRequest

__all__ = ['CardExtractor', 'SingleCardExtractor', 'Card']

class CardExtractor(object):
    """Extracts card information from Gatherer HTML."""

    def __init__(self, html):
        self.html = html

    def _group(self, lst, n):
        newlist = []
        for i in range(0, len(lst), n):
            val = lst[i:i+n]
            if len(val) == n:
                newlist.append(tuple(val))
        return newlist

    def _flatten(self, element):
        """Recursively enter and extract text from all child
        elements."""
        result = [ (element.text or '') ]
        if element.attrib.get('alt'):
            result.append(Symbol(element.attrib.get('alt')).textbox)
        for sel in element:
            result.append(self._flatten(sel))
            result.append(sel.tail or '')
        return ''.join(result)

    def extract_many(self):
        doc = parse(self.html).getroot()
        cards = []
        for item in doc.cssselect('tr.cardItem'):
            for c in item.cssselect('div.cardInfo'):
                card = Card()
                card.name = c.cssselect('span.cardTitle')[0].text_content().strip()
                for img in c.cssselect('span.manaCost img'):
                    setattr(card, 'mana_cost', Symbol(img.attrib['alt']).short)
                regex = '\([^/]+/[^)]+\)'
                typeline = c.cssselect('span.typeLine')[0].text_content()
                m = re.search(regex, typeline)
                if m:
                    card.pow_tgh = m.group(0)
                    card.types = re.sub(regex, '', typeline).strip()
                else:
                    card.types = typeline.strip()
                card.types = card.types.replace(u'\xe2\x80\x94', u'\u2014')
                card.card_text = self._flatten(c.cssselect('div.rulesText')[0]).strip()

            set_rarity = ', '.join([img.attrib['alt'] for img in
                                    item.cssselect('td.rightCol img')])
            setattr(card, 'set_rarity', set_rarity)
            cards.append(card)
        return cards

    def extract(self, get_card_urls=False):
        if not self.html:
            return False

        doc = parse(self.html).getroot()
        cards = []
        for component in doc.cssselect('td.cardComponentContainer'):
            if not component.getchildren():
                continue # do not parse empty components
            labels = component.cssselect('div.label')
            values = component.cssselect('div.value')
            pairs = zip(labels, values)
            card = Card()
            for (label, value) in pairs:
                l = label.text_content().strip().replace(' ', '_') \
                    .replace(':', '').lower().replace('#', 'number')

                if l == 'card_text':
                    v = ' ; '.join(map(self._flatten,
                                       value.cssselect('div.cardtextbox')))
                else:
                    v = u''
                    if l == 'mana_cost':
                        for img in value.cssselect('img'):
                            v += Symbol(img.attrib['alt']).short
                    v += value.text_content().strip()
                setattr(card, l, v.replace(u'\xe2\x80\x94', u'\u2014'))
            cards.append(card)
        return cards



        # soup = BeautifulSoup.BeautifulSoup(self.html)
        # if not soup.table:
        #     return []
        # for tag in soup.findAll('br'):
        #     tag.replaceWith('||')

        # td_tags = soup.table.findAll('td')

        # # Get rulings hrefs here.
        # if get_card_urls:
        #     a_tags = soup.table.findAll('a')
        #     card_urls = [tag['href'] for tag in a_tags]

        # content_lists = [tag.contents for tag in td_tags]
        # unified_content = []
        # cards = []
        # for lst in content_lists:
        #     unified_content.append(''.join([item.string or u'' for item in lst]))

        # unified_content = [item for item in unified_content if item != u'\n||\n']
        # unified_content = self._group(unified_content, 2)

        # blocks  = []
        # block = []
        # for u in unified_content:
        #     block.append(u)
        #     if 'Set/Rarity' in u[0]:
        #         blocks.append(block)
        #         block = []

        # for block in blocks:
        #     card = Card.from_block(block)
        #     if get_card_urls:
        #         card.url = card_urls.pop(0)
        #     cards.append(card)
        # return cards

class Symbol(object):

    def __init__(self, text):
        self.text = text
        self.specials = {'Untap': 'Q',
                         'Blue': 'U',
                         'Snow': 'S}i',
                         'Variable Colorless': 'X'}

    @property
    def short(self):
        if self.text in self.specials.keys():
            return self.specials[self.text]
        elif self.is_hybrid:
            return self.hybrid
        elif self.is_phyrexian:
            return self.phyrexian
        elif self.text.isdigit():
            return self.text
        return self.text[:1]

    @property
    def is_phyrexian(self):
        return 'Phyrexian ' in self.text

    @property
    def phyrexian(self):
        return '({0}/P)'.format(
            Symbol(self.text.replace('Phyrexian ', '')).short)

    @property
    def is_hybrid(self):
        return ' or ' in self.text

    @property
    def hybrid(self):
        return '({0})'.format('/'.join(
                Symbol(l).short for l in self.text.split(' or ')))

    @property
    def textbox(self):
        base = '{{{0}}}'.format(self.short)
        if '/' in base: return base.lower()
        else: return base


class SingleCardExtractor(object):

    def __init__(self, html):
        self.html = html

    def _parse_html(self):
        if not self.html:
            return False

    def extract_flavor(self):
{'fraction_non_alphanumeric': 0.1330082394484614, 'fraction_numerical': 0.005044560282495376, 'mean_word_length': 1.6304732419283503, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '481853', 'n_tokens_mistral': 3574, 'n_tokens_neox': 3421, 'n_words': 768}