<commit_before>
import re

from html.entities import codepoint2name, name2codepoint

from lxml import etree


# Base class

class SomewhatObjectifiedElement(etree.ElementBase):
    nsmap = {'cs': 'http://purl.org/net/xbiblio/csl',
             'xml': 'http://www.w3.org/XML/1998/namespace'}

    # TODO: what about multiple instances of the same name?
    def __getattr__(self, name):
        return self.find('cs:' + name, self.nsmap)


class CitationStylesElement(SomewhatObjectifiedElement):
    _default_options = {# global options
                        'initialize-with-hyphen': 'true',
                        'page-range-format': None,
                        'demote-non-dropping-particle': 'display-and-sort',

                        # inheritable name(s) options
                        'and': None,
                        'delimiter-precedes-et-al': 'contextual',
                        'delimiter-precedes-last': 'contextual',
                        'et-al-min': 0,
                        'et-al-use-first': 1,
                        'et-al-subsequent-min': 0,
                        'et-al-subsequent-use-first': 1,
                        'et-al-use-last': 'false',
                        'initialize-with': None,
                        'name-as-sort-order': None,
                        'sort-separator': ', ',

                        'name-form': 'long',
                        'name-delimiter': ', ',
                        'names-delimiter': ''}

    def get_root(self):
        return self.getroottree().getroot()

    def xpath_search(self, expression):
        return self.xpath(expression, namespaces=self.nsmap)

    def get_option(self, name):
        return self.get(name, __class__._default_options[name])

    def get_citation(self):
        return self.get_root().citation

    def get_bibliography(self):
        return self.get_root().bibliography

    def get_macro(self, name):
        expression = "cs:macro[@name='{}'][1]".format(name)
        return self.get_root().xpath_search(expression)[0]

    def get_layout(self):
        return self.xpath_search('./ancestor::cs:layout[1]')[0]

    # TODO: Locale methods
    def get_term(self, name, form=None):
        if isinstance(self.get_root(), Locale):
            return self.get_root().get_term(name, form)
        else:
            for locale in self.get_root().locales:
                try:
                    return locale.get_term(name, form)
                except IndexError: # TODO: create custom exception
                    continue

    def get_date(self, form):
        for locale in self.get_root().locales:
            try:
                return locale.get_date(form)
            except IndexError:
                continue

    def get_locale_option(self, name):
        for locale in self.get_root().locales:
            try:
                return locale.get_option(name)
            except IndexError:
                continue


# Top level elements

class Style(CitationStylesElement):
    def set_locale_list(self, system_locale):
        from . import CitationStylesLocale

        def search_locale(locale):
            return self.xpath_search('./cs:locale[@xml:lang="{}"]'
                                     .format(locale))[0]

        self.locales = []
        try:
            self.locales.append(search_locale(system_locale))
        except IndexError:
            pass

        language = system_locale.split('-')[0]
        try:
            self.locales.append(search_locale(language))
        except IndexError:
            pass

        try:
            expr = './cs:locale[not(@xml:lang)]'
            self.locales.append(self.xpath_search(expr)[0])
        except IndexError:
            pass

        self.locales.append(CitationStylesLocale(system_locale).root)
        if system_locale != 'en-US':
            # TODO: add other locales with same locale
            self.locales.append(CitationStylesLocale('en-US').root)


class Locale(CitationStylesElement):
    _default_options = {'punctuation-in-quote': 'false'}

    def get_term(self, name, form=None):
        attributes = "@name='{}'".format(name)
        if form is not None:
            attributes += " and @form='{}'".format(form)
        else:
            attributes += " and not(@form)"
        expr = './cs:term[{}]'.format(attributes)
        try:
            return self.terms.xpath_search(expr)[0]
        except AttributeError:
            raise IndexError

    def get_date(self, form):
        expr = "./cs:date[@form='{}']".format(form)
        return self.xpath_search(expr)[0]

    def get_option(self, name):
        options = self.find('cs:style-options', self.nsmap)
        if options is None:
            raise IndexError
        return options.get(name, __class__._default_options[name])


class FormattingInstructions(object):
    def get_option(self, name):
        if name in self._default_options:
            return self.get(name, self._default_options[name])
        else:
            return self.get(name, self.get_root().get_option(name))

    def render(self, reference):
        raise NotImplementedError


class Citation(FormattingInstructions, CitationStylesElement):
    _default_options = {# disambiguation
                        'disambiguate-add-names': False,
                        'disambiguate-add-givenname': False,
                        'givenname-disambiguation-rule': 'all-names',
                        'disambiguate-add-year-suffix': False,

                        # citation collapsing
                        'collapse': None,
                        'year-suffix-delimiter': None,
                        'after-collapse-delimiter': None,

                        # note distance
                        'near-note-distance': 5}

    def render(self, citation):
        return self.layout.render_citation(citation)


class Bibliography(FormattingInstructions, CitationStylesElement):
    _default_options = {# whitespace
                        'hanging-indent': False,
                        'second-field-align': None,
                        'line-spacing': 1,
                        'entry-spacing': 1,

                        # reference grouping
                        'subsequent-author-substitute': None}

    def render(self, reference):
        return self.layout.render_bibliography(reference)


# Style behavior

class Formatted(object):
    def format(self, string):
        text = self.font_style(string)
        text = self.font_variant(text)
        text = self.font_weight(text)
        text = self.text_decoration(text)
        text = self.vertical_align(text)
        return text

    def font_style(self, text):
        font_style = self.get('font-style', 'normal')
        if font_style == 'normal':
            wrappers = '', ''
        elif font_style == 'italic':
            wrappers = '<i>', '</i>'
        elif font_style == 'oblique':
            wrappers = '<i>', '</i>'
        return '{1}{0}{2}'.format(text, *wrappers)

    def font_variant(self, text):
        font_variant = self.get('font-variant', 'normal')
        if font_variant == 'normal':
            wrappers = '', ''
        elif font_variant == 'small-caps':
            wrappers = '<span style="font-variant:small-caps;">', '</span>'
        return '{1}{0}{2}'.format(text, *wrappers)

    def font_weight(self, text):
        font_weight = self.get('font-weight', 'normal')
        if font_weight == 'normal':
            wrappers = '', ''
        elif font_weight == 'bold':
            wrappers = '<b>', '</b>'
        elif font_weight == 'light':
            wrappers = '<l>', '</l>'
        return '{1}{0}{2}'.format(text, *wrappers)

    def text_decoration(self, text):
        text_decoration = self.get('text-decoration', 'none')
        if text_decoration == 'none':
            wrappers = '', ''
        elif text_decoration == 'underline':
            wrappers = '<u>', '</u>'
        return '{1}{0}{2}'.format(text, *wrappers)

    def vertical_align(self, text):
        vertical_align = self.get('vertical-align', 'baseline')
        if vertical_align == 'baseline':
            wrappers = '', ''
        elif vertical_align == 'sup':
            wrappers = '<sup>', '</sup>'
        elif vertical_align == 'sub':
            wrappers = '<sub>', '</sub>'
        return '{1}{0}{2}'.format(text, *wrappers)


class Affixed(object):
    def wrap(self, string):
        prefix = self.get('prefix', '')
        suffix = self.get('suffix', '')
        return prefix + string + suffix


class Delimited(object):
    def join(self, strings, default_delimiter=''):
        delimiter = self.get('delimiter', default_delimiter)
        return delimiter.join([item for item in strings if item is not None])


class Displayed(object):
    pass


class Quoted(object):
    def quote(self, string):
        piq = self.get_locale_option('punctuation-in-quote').lower() == 'true'
        if self.get('quotes', 'false').lower() == 'true':
            open_quote = self.get_term('open-quote').single
            close_quote = self.get_term('close-quote').single
            string = open_quote + string + close_quote
##            quoted_string = QuotedString(string, open_quote, close_quote, piq)
        return string


class StrippedPeriods(object):
    def strip_periods(self, string):
        strip_periods = self.get('strip-periods', 'false').lower() == 'true'
        if strip_periods:
            string = string.replace('.', '')
        return string


class TextCased(object):
    def case(self, text):
        text_case = self.get('text-case')
        if text_case is not None:
            if text_case == 'lowercase':
                text = text.lower()
            elif text_case == 'uppercase':
                text = text.upper()
            elif text_case == 'capitalize-first':
                text = text.capitalize()
            elif text_case == 'capitalize-all':
                text = text.title()
            elif text_case == 'title':
                text = text.title()
            elif text_case == 'sentence':
                text = text.capitalize()
        return text


# Locale

class Term(CitationStylesElement):
    @property
    def single(self):
        try:
            return self.find('cs:single', self.nsmap).text
        except AttributeError:
            return self.text or ''

    @property
    def multiple(self):
        try:
            return self.find('cs:multiple', self.nsmap).text
        except AttributeError:
            return self.text or ''


# Rendering elements

class Parent(object):
    def calls_variable(self):
        return any([child.calls_variable() for child in self.getchildren()])

    def render_children(self, item, *args, **kwargs):
        from ...bibliography import VariableError
        output = []
        for child in self.iterchildren():
            try:
                text = child.render(item, *args, **kwargs)
                if text is not None:
                    output.append(text)
            except VariableError:
                pass
        if output:
            return ''.join(output)
        else:
            return None


class Sort(CitationStylesElement):
    def sort(self, items):
        raise NotImplementedError
        sort_keys = []
        for key in self.key:
            try:
                descending = key.attrib['sort'] == 'descending'
            except KeyError:
                descending = False
            # names-min, names-use-first
            sort_keys.append((key.variable, descending))


class Macro(CitationStylesElement, Parent):
    def render(self, item, context):
        return self.render_children(item, context=context)


class Layout(CitationStylesElement, Parent, Formatted, Affixed, Delimited):
    def render_citation(self, citation):
        from ...bibliography import VariableError
        # TODO: formatting
        self.repressed = {}
        out = []
        for item in citation.items:
            prefix = item.get('prefix', '')
            suffix = item.get('suffix', '')
            try:
                output = self.render_children(item)
                if output is not None:
                    text = prefix + output + suffix
                    out.append(text)
            except VariableError:
                pass
            self.repressed = {}
        return self.format(self.wrap(self.join(out)))

    def render_bibliography(self, references):
        from ...bibliography import CitationItem
        self.repressed = {}
        output = ['<div class="csl-bib-body">']
        items = [CitationItem(reference) for reference in references]
        for item in items:
            text = self.render_children(item)
            if text is not None:
                text = '  <div class="csl-entry">' + text + '</div>'
                output.append(text)
            self.repressed = {}
        output.append('</div>')
        return '\n'.join(output)


class Text(CitationStylesElement, Formatted, Affixed, Quoted, TextCased,
           StrippedPeriods):
    def calls_variable(self):
        return 'variable' in self.attrib

    def render(self, item, context=None):
        if context is None:
            context = self

        if 'variable' in self.attrib:
            variable = self.get('variable')
            repressed = context.get_layout().repressed
            if self.tag in repressed and variable in repressed[self.tag]:
                return None

            if self.get('form') == 'short':
                short_variable = variable + '-short'
                if short_variable.replace('-', '_') in item.reference:
                    variable = short_variable

            if variable == 'page-first' and variable not in item.reference:
                page = item.reference.page
                text = Number.re_range.match(page).group(1)
            else:
                text = item.reference[variable.replace('-', '_')]
        elif 'macro' in self.attrib:
            text = self.get_macro(self.get('macro')).render(item, context)
        elif 'term' in self.attrib:
            form = self.get('form', 'long')
            plural = self.get('plural', 'false').lower() == 'true'
            if form == 'long':
                form = None
            term = self.get_term(self.get('term'), form)<commit_msg>fixed calls_variable for text calling a macro<commit_after>
import re

from html.entities import codepoint2name, name2codepoint

from lxml import etree


# Base class

class SomewhatObjectifiedElement(etree.ElementBase):
    nsmap = {'cs': 'http://purl.org/net/xbiblio/csl',
             'xml': 'http://www.w3.org/XML/1998/namespace'}

    # TODO: what about multiple instances of the same name?
    def __getattr__(self, name):
        return self.find('cs:' + name, self.nsmap)


class CitationStylesElement(SomewhatObjectifiedElement):
    _default_options = {# global options
                        'initialize-with-hyphen': 'true',
                        'page-range-format': None,
                        'demote-non-dropping-particle': 'display-and-sort',

                        # inheritable name(s) options
                        'and': None,
                        'delimiter-precedes-et-al': 'contextual',
                        'delimiter-precedes-last': 'contextual',
                        'et-al-min': 0,
                        'et-al-use-first': 1,
                        'et-al-subsequent-min': 0,
                        'et-al-subsequent-use-first': 1,
                        'et-al-use-last': 'false',
                        'initialize-with': None,
                        'name-as-sort-order': None,
                        'sort-separator': ', ',

                        'name-form': 'long',
                        'name-delimiter': ', ',
                        'names-delimiter': ''}

    def get_root(self):
        return self.getroottree().getroot()

    def xpath_search(self, expression):
        return self.xpath(expression, namespaces=self.nsmap)

    def get_option(self, name):
        return self.get(name, __class__._default_options[name])

    def get_citation(self):
        return self.get_root().citation

    def get_bibliography(self):
        return self.get_root().bibliography

    def get_macro(self, name):
        expression = "cs:macro[@name='{}'][1]".format(name)
        return self.get_root().xpath_search(expression)[0]

    def get_layout(self):
        return self.xpath_search('./ancestor::cs:layout[1]')[0]

    # TODO: Locale methods
    def get_term(self, name, form=None):
        if isinstance(self.get_root(), Locale):
            return self.get_root().get_term(name, form)
        else:
            for locale in self.get_root().locales:
                try:
                    return locale.get_term(name, form)
                except IndexError: # TODO: create custom exception
                    continue

    def get_date(self, form):
        for locale in self.get_root().locales:
            try:
                return locale.get_date(form)
            except IndexError:
                continue

    def get_locale_option(self, name):
        for locale in self.get_root().locales:
            try:
                return locale.get_option(name)
            except IndexError:
                continue


# Top level elements

class Style(CitationStylesElement):
    def set_locale_list(self, system_locale):
        from . import CitationStylesLocale

        def search_locale(locale):
            return self.xpath_search('./cs:locale[@xml:lang="{}"]'
                                     .format(locale))[0]

        self.locales = []
        try:
            self.locales.append(search_locale(system_locale))
        except IndexError:
            pass

        language = system_locale.split('-')[0]
        try:
            self.locales.append(search_locale(language))
        except IndexError:
            pass

        try:
            expr = './cs:locale[not(@xml:lang)]'
            self.locales.append(self.xpath_search(expr)[0])
        except IndexError:
            pass

        self.locales.append(CitationStylesLocale(system_locale).root)
        if system_locale != 'en-US':
            # TODO: add other locales with same locale
            self.locales.append(CitationStylesLocale('en-US').root)


class Locale(CitationStylesElement):
    _default_options = {'punctuation-in-quote': 'false'}

    def get_term(self, name, form=None):
        attributes = "@name='{}'".format(name)
        if form is not None:
            attributes += " and @form='{}'".format(form)
        else:
            attributes += " and not(@form)"
        expr = './cs:term[{}]'.format(attributes)
        try:
            return self.terms.xpath_search(expr)[0]
        except AttributeError:
            raise IndexError

    def get_date(self, form):
        expr = "./cs:date[@form='{}']".format(form)
        return self.xpath_search(expr)[0]

    def get_option(self, name):
        options = self.find('cs:style-options', self.nsmap)
        if options is None:
            raise IndexError
        return options.get(name, __class__._default_options[name])


class FormattingInstructions(object):
    def get_option(self, name):
        if name in self._default_options:
            return self.get(name, self._default_options[name])
        else:
            return self.get(name, self.get_root().get_option(name))

    def render(self, reference):
        raise NotImplementedError


class Citation(FormattingInstructions, CitationStylesElement):
    _default_options = {# disambiguation
                        'disambiguate-add-names': False,
                        'disambiguate-add-givenname': False,
                        'givenname-disambiguation-rule': 'all-names',
                        'disambiguate-add-year-suffix': False,

                        # citation collapsing
                        'collapse': None,
                        'year-suffix-delimiter': None,
                        'after-collapse-delimiter': None,

                        # note distance
                        'near-note-distance': 5}

    def render(self, citation):
        return self.layout.render_citation(citation)


class Bibliography(FormattingInstructions, CitationStylesElement):
    _default_options = {# whitespace
                        'hanging-indent': False,
                        'second-field-align': None,
                        'line-spacing': 1,
                        'entry-spacing': 1,

                        # reference grouping
                        'subsequent-author-substitute': None}

    def render(self, reference):
        return self.layout.render_bibliography(reference)


# Style behavior

class Formatted(object):
    def format(self, string):
        text = self.font_style(string)
        text = self.font_variant(text)
        text = self.font_weight(text)
        text = self.text_decoration(text)
        text = self.vertical_align(text)
        return text

    def font_style(self, text):
        font_style = self.get('font-style', 'normal')
        if font_style == 'normal':
            wrappers = '', ''
        elif font_style == 'italic':
            wrappers = '<i>', '</i>'
        elif font_style == 'oblique':
            wrappers = '<i>', '</i>'
        return '{1}{0}{2}'.format(text, *wrappers)

    def font_variant(self, text):
        font_variant = self.get('font-variant', 'normal')
        if font_variant == 'normal':
            wrappers = '', ''
        elif font_variant == 'small-caps':
            wrappers = '<span style="font-variant:small-caps;">', '</span>'
        return '{1}{0}{2}'.format(text, *wrappers)

    def font_weight(self, text):
        font_weight = self.get('font-weight', 'normal')
        if font_weight == 'normal':
            wrappers = '', ''
        elif font_weight == 'bold':
            wrappers = '<b>', '</b>'
        elif font_weight == 'light':
            wrappers = '<l>', '</l>'
        return '{1}{0}{2}'.format(text, *wrappers)

    def text_decoration(self, text):
        text_decoration = self.get('text-decoration', 'none')
        if text_decoration == 'none':
            wrappers = '', ''
        elif text_decoration == 'underline':
            wrappers = '<u>', '</u>'
        return '{1}{0}{2}'.format(text, *wrappers)

    def vertical_align(self, text):
        vertical_align = self.get('vertical-align', 'baseline')
        if vertical_align == 'baseline':
            wrappers = '', ''
        elif vertical_align == 'sup':
            wrappers = '<sup>', '</sup>'
        elif vertical_align == 'sub':
            wrappers = '<sub>', '</sub>'
        return '{1}{0}{2}'.format(text, *wrappers)


class Affixed(object):
    def wrap(self, string):
        prefix = self.get('prefix', '')
        suffix = self.get('suffix', '')
        return prefix + string + suffix


class Delimited(object):
    def join(self, strings, default_delimiter=''):
        delimiter = self.get('delimiter', default_delimiter)
        return delimiter.join([item for item in strings if item is not None])


class Displayed(object):
    pass


class Quoted(object):
    def quote(self, string):
        piq = self.get_locale_option('punctuation-in-quote').lower() == 'true'
        if self.get('quotes', 'false').lower() == 'true':
            open_quote = self.get_term('open-quote').single
            close_quote = self.get_term('close-quote').single
            string = open_quote + string + close_quote
##            quoted_string = QuotedString(string, open_quote, close_quote, piq)
        return string


class StrippedPeriods(object):
    def strip_periods(self, string):
        strip_periods = self.get('strip-periods', 'false').lower() == 'true'
        if strip_periods:
            string = string.replace('.', '')
        return string


class TextCased(object):
    def case(self, text):
        text_case = self.get('text-case')
        if text_case is not None:
            if text_case == 'lowercase':
                text = text.lower()
            elif text_case == 'uppercase':
                text = text.upper()
            elif text_case == 'capitalize-first':
                text = text.capitalize()
            elif text_case == 'capitalize-all':
                text = text.title()
            elif text_case == 'title':
                text = text.title()
            elif text_case == 'sentence':
                text = text.capitalize()
        return text


# Locale

class Term(CitationStylesElement):
    @property
    def single(self):
        try:
            return self.find('cs:single', self.nsmap).text
        except AttributeError:
            return self.text or ''

    @property
    def multiple(self):
        try:
            return self.find('cs:multiple', self.nsmap).text
        except AttributeError:
            return self.text or ''


# Rendering elements

class Parent(object):
    def calls_variable(self):
        return any([child.calls_variable() for child in self.getchildren()])

    def render_children(self, item, *args, **kwargs):
        from ...bibliography import VariableError
        output = []
        for child in self.iterchildren():
            try:
                text = child.render(item, *args, **kwargs)
                if text is not None:
                    output.append(text)
            except VariableError:
                pass
        if output:
            return ''.join(output)
        else:
            return None


class Sort(CitationStylesElement):
    def sort(self, items):
        raise NotImplementedError
        sort_keys = []
        for key in self.key:
            try:
                descending = key.attrib['sort'] == 'descending'
            except KeyError:
                descending = False
            # names-min, names-use-first
            sort_keys.append((key.variable, descending))


class Macro(CitationStylesElement, Parent):
    def render(self, item, context):
        return self.render_children(item, context=context)


class Layout(CitationStylesElement, Parent, Formatted, Affixed, Delimited):
    def render_citation(self, citation):
        from ...bibliography import VariableError
        # TODO: formatting
        self.repressed = {}
        out = []
        for item in citation.items:
            prefix = item.get('prefix', '')
            suffix = item.get('suffix', '')
            try:
                output = self.render_children(item)
                if output is not None:
                    text = prefix + output + suffix
                    out.append(text)
            except VariableError:
                pass
            self.repressed = {}
        return self.format(self.wrap(self.join(out)))

    def render_bibliography(self, references):
        from ...bibliography import CitationItem
        self.repressed = {}
        output = ['<div class="csl-bib-body">']
        items = [CitationItem(reference) for reference in references]
        for item in items:
            text = self.render_children(item)
            if text is not None:
                text = '  <div class="csl-entry">' + text + '</div>'
                output.append(text)
            self.repressed = {}
        output.append('</div>')
        return '\n'.join(output)


class Text(CitationStylesElement, Formatted, Affixed, Quoted, TextCased,
           StrippedPeriods):
    def calls_variable(self):
        if 'variable' in self.attrib:
            return True
        elif 'macro' in self.attrib:
            return self.get_macro(self.get('macro')).calls_variable()
        else:
            return False

    def render(self, item, context=None):
        if context is None:
            context = self

        if 'variable' in self.attrib:
            variable = self.get('variable')
            repressed = context.get_layout().repressed
            if self.tag in repressed and variable in repressed[self.tag]:
                return None

            if self.get('form') == 'short':
                short_variable = variable + '-short'
                if short_variable.replace('-', '_') in item.reference:
                    variable = short_variable

            if variable == 'page-first' and variable not in item.reference:
                page = item.reference.page
                text = Number.re_range.match(page).group(1)
            else:
                text = item.reference[variable.replace('-', '_')]
        elif 'macro' in self.attrib:
            text = self.get_macro(self.get('macro')).render(item, context)
        elif 'term' in self.attrib:
            form = self.get('form', 'long')
            plural = self.get('plural', 'false').lower() == 'true'
            if form == 'long':
                form = None
            term = self.get_term(self.get('term'), form)
{'fraction_non_alphanumeric': 0.12159307359307359, 'fraction_numerical': 0.0027012987012987014, 'mean_word_length': 1.8743778618355564, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2070781', 'n_tokens_mistral': 8116, 'n_tokens_neox': 7753, 'n_words': 1776}