<commit_before>"""Test the actual conversion of the text inside an Android xml file
and a gettext .po file; since the rules of both formats differ, the
actual characters/bytes of each version will differ, while still
representing the same localizable message.

Tests in here ensure that this conversation happens correctly; that is,
as closely to the way Android itself processes it's format as possible,
without modifying a string (in terms of significant content).

This currently follows the behavior I was seeing on an emulator using the
Android 2.0 SDK release.
"""

import re
from StringIO import StringIO
from lxml import etree
from babel.messages import Catalog
from android2po import xml2po, po2xml


class TestFromXML():
    """Test reading from Android's XML format.
    """

    @classmethod
    def assert_convert(cls, xml, po=None):
        """Helper that passes the string in ``xml`` through our xml
        parsing mechanism, and checks the resulting po catalog string
        against ``po``.

        If ``po`` is not given, we check against ``xml`` instead, i.e.
        expect the string to remain unchanged.
        """
        key = 'test'
        catalog = xml2po(StringIO(
            '<resources><string name="%s">%s</string></resources>' % (key, xml)))
        match = po if po is not None else xml
        for message in catalog:
            if message.context == key:
                #print "'%s' == '%s'" % (message.id, match)
                print repr(message.id), '==', repr(match)
                assert message.id == match
                break
        else:
            raise KeyError()

    def test_basic(self):
        """Test some basic string variations.
        """
        # No nested tags.
        self.assert_convert('bar')
        # Nested tags only.
        self.assert_convert('<b>world</b>')
        # [bug] Trailing text + nested tags.
        self.assert_convert('hello <b>world</b>')
        # Multiple levels of nesting.
        self.assert_convert('<b><u>hello</u> world</b>')

    def test_tags_and_attributes(self):
        """Test certain XML-inherited syntax elements, in particular,
        that attributes of nested tags are rendered properly.

        I haven't actually tested if Android even supports them, but
        there should be no damage from our side in persisting them.
        If they, say, aren't allowed, the developer will have to deal
        with it anyway.
        """
        self.assert_convert('<b name="">foo</b>')
        # Order is persisted.
        self.assert_convert('<b k1="1" k2="2" k3="3">foo</b>')
        # Quotes are normalized.
        self.assert_convert('<b k2=\'2\'>foo</b>', '<b k2="2">foo</b>')

        # Since we can't know whether a tag was self-closing, such
        # tags are going to be expanded when going through us.
        self.assert_convert('<b />', '<b></b>')

    def test_whitespace(self):
        """Test various whitespace handling scenarios.
        """
        # Intermediate whitespace is collapsed.
        self.assert_convert('a      b       c', 'a b c')
        # Leading and trailing whitespace is removed completely only
        # if no tags are nested.
        self.assert_convert('    a  ', 'a')
        # If there are nested tags, normal whitespace collapsing rules
        # apply at the beginning and end of the string instead.
        self.assert_convert('    <b></b>  ', ' <b></b> ')
        # Whitespace collapsing does not reach beyond a nested tag, i.e.
        # each text between two tags manages it's whitespace independently.
        self.assert_convert('   <b>   <u>    </u>  </b>  ', ' <b> <u> </u> </b> ')

        # Newlines and even tabs are considered whitespace as well.
        self.assert_convert('a    \n\n   \n   \n\n  b', 'a b')
        self.assert_convert('a  \t\t   \t  b', 'a b')
        # [bug] Edge case in which a non-significant newline/tab used to
        # end up in the output (when the last whitespace character was
        # such a newline or tab (or other whitespace other than 'space').
        self.assert_convert('a\n\n\nb', 'a b')
        self.assert_convert('a\t\t\tb', 'a b')

        # Quoting protects whitespace.
        self.assert_convert('"    a     b    "', '    a     b    ')

    def test_escaping(self):
        """Test escaping.
        """
        self.assert_convert(r'new\nline',  'new\nline')
        self.assert_convert(r'foo:\tbar',  'foo:\tbar')
        self.assert_convert(r'my name is \"earl\"',  'my name is "earl"')
        self.assert_convert(r'my name is \'earl\'',  'my name is \'earl\'')
        self.assert_convert(r'\\',  '\\')

        # Test a practical case of a double-backslash protecting an
        # escape sequence.
        self.assert_convert(r'\\n',  r'\n')

        # XXX: Android seems to even normalize inserted newlines:
        #    r'\n\n\n\n\n' (as a literal string) ends up as ''
        #    r'a\n\n\n\n\n' (as a literal string) ends up as 'a'
        #    r'a\n\n\n\n\nb' (as a literal string) ends up as 'a\nb'
        # It doesn't do the same for tabs:
        #    r'a\t\t\t\tb' has the tabs included
        # Actually! This only seems to be the case when you output the
        # string using the log; setting it to the caption of a textview,
        # for example, keeps the multiple linebreaks.

        # A double slash can be used to protect escapes.
        self.assert_convert(r'new\\nline',  'new\\nline')<commit_msg>Added another test for whitespace-only strings.<commit_after>"""Test the actual conversion of the text inside an Android xml file
and a gettext .po file; since the rules of both formats differ, the
actual characters/bytes of each version will differ, while still
representing the same localizable message.

Tests in here ensure that this conversation happens correctly; that is,
as closely to the way Android itself processes it's format as possible,
without modifying a string (in terms of significant content).

This currently follows the behavior I was seeing on an emulator running
the Android 1.6 SDK release, with an application compiled against the
2.0 SDK release.
"""

import re
from StringIO import StringIO
from lxml import etree
from babel.messages import Catalog
from nose.tools import assert_raises
from android2po import xml2po, po2xml


class TestFromXML():
    """Test reading from Android's XML format.
    """

    @classmethod
    def assert_convert(cls, xml, po=None):
        """Helper that passes the string in ``xml`` through our xml
        parsing mechanism, and checks the resulting po catalog string
        against ``po``.

        If ``po`` is not given, we check against ``xml`` instead, i.e.
        expect the string to remain unchanged.
        """
        key = 'test'
        catalog = xml2po(StringIO(
            '<resources><string name="%s">%s</string></resources>' % (key, xml)))
        match = po if po is not None else xml
        for message in catalog:
            if message.context == key:
                #print "'%s' == '%s'" % (message.id, match)
                print repr(message.id), '==', repr(match)
                assert message.id == match
                break
        else:
            raise KeyError()

    def test_basic(self):
        """Test some basic string variations.
        """
        # No nested tags.
        self.assert_convert('bar')
        # Nested tags only.
        self.assert_convert('<b>world</b>')
        # [bug] Trailing text + nested tags.
        self.assert_convert('hello <b>world</b>')
        # Multiple levels of nesting.
        self.assert_convert('<b><u>hello</u> world</b>')

    def test_tags_and_attributes(self):
        """Test certain XML-inherited syntax elements, in particular,
        that attributes of nested tags are rendered properly.

        I haven't actually tested if Android even supports them, but
        there should be no damage from our side in persisting them.
        If they, say, aren't allowed, the developer will have to deal
        with it anyway.
        """
        self.assert_convert('<b name="">foo</b>')
        # Order is persisted.
        self.assert_convert('<b k1="1" k2="2" k3="3">foo</b>')
        # Quotes are normalized.
        self.assert_convert('<b k2=\'2\'>foo</b>', '<b k2="2">foo</b>')

        # Since we can't know whether a tag was self-closing, such
        # tags are going to be expanded when going through us.
        self.assert_convert('<b />', '<b></b>')

    def test_whitespace(self):
        """Test various whitespace handling scenarios.
        """
        # Intermediate whitespace is collapsed.
        self.assert_convert('a      b       c', 'a b c')
        # Leading and trailing whitespace is removed completely only
        # if no tags are nested.
        self.assert_convert('    a  ', 'a')
        # If there are nested tags, normal whitespace collapsing rules
        # apply at the beginning and end of the string instead.
        self.assert_convert('    <b></b>  ', ' <b></b> ')
        # Whitespace collapsing does not reach beyond a nested tag, i.e.
        # each text between two tags manages it's whitespace independently.
        self.assert_convert('   <b>   <u>    </u>  </b>  ', ' <b> <u> </u> </b> ')

        # Newlines and even tabs are considered whitespace as well.
        self.assert_convert('a    \n\n   \n   \n\n  b', 'a b')
        self.assert_convert('a  \t\t   \t  b', 'a b')
        # [bug] Edge case in which a non-significant newline/tab used to
        # end up in the output (when the last whitespace character was
        # such a newline or tab (or other whitespace other than 'space').
        self.assert_convert('a\n\n\nb', 'a b')
        self.assert_convert('a\t\t\tb', 'a b')

        # An all whitespace string isn't even included.
        assert_raises(KeyError, self.assert_convert, '   ', '')

        # Quoting protects whitespace.
        self.assert_convert('"    a     b    "', '    a     b    ')

    def test_escaping(self):
        """Test escaping.
        """
        self.assert_convert(r'new\nline',  'new\nline')
        self.assert_convert(r'foo:\tbar',  'foo:\tbar')
        self.assert_convert(r'my name is \"earl\"',  'my name is "earl"')
        self.assert_convert(r'my name is \'earl\'',  'my name is \'earl\'')
        self.assert_convert(r'\\',  '\\')

        # Test a practical case of a double-backslash protecting an
        # escape sequence.
        self.assert_convert(r'\\n',  r'\n')

        # XXX: Android seems to even normalize inserted newlines:
        #    r'\n\n\n\n\n' (as a literal string) ends up as ''
        #    r'a\n\n\n\n\n' (as a literal string) ends up as 'a'
        #    r'a\n\n\n\n\nb' (as a literal string) ends up as 'a\nb'
        # It doesn't do the same for tabs:
        #    r'a\t\t\t\tb' has the tabs included
        # Actually! This only seems to be the case when you output the
        # string using the log; setting it to the caption of a textview,
        # for example, keeps the multiple linebreaks.

        # A double slash can be used to protect escapes.
        self.assert_convert(r'new\\nline',  'new\\nline')
{'fraction_non_alphanumeric': 0.11448053113224475, 'fraction_numerical': 0.003050421675937556, 'mean_word_length': 2.28917084685748, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3580409', 'n_tokens_mistral': 3300, 'n_tokens_neox': 3184, 'n_words': 1208}