<commit_before>from __future__ import absolute_import, unicode_literals
import six


class BaseField(object):
    converter = None

    def __init__(self, name, default=None):
        self.name = name
        self.default = default

    def to_xml(self):
        raise NotImplementedError()


class StringField(BaseField):
    converter = six.text_type

    def to_xml(self, value):
        value = self.converter(value)
        tpl = '<{name}>![CDATA[{value}]]</{name}>'
        return tpl.format(name=self.name, value=value)


class IntegerField(BaseField):
    converter = int

    def to_xml(self, value):
        value = self.converter(value)
        tpl = '<{name}>{value}</{name}>'
        return tpl.format(name=self.name, value=value)


class FloatField(BaseField):
    converter = float

    def to_xml(self, value):
        value = self.converter(value)
        tpl = '<{name}>{value}</{name}>'
        return tpl.format(name=self.name, value=value)<commit_msg>Add some more fields for replies<commit_after>from __future__ import absolute_import, unicode_literals
import six


class BaseField(object):
    converter = None

    def __init__(self, name, default=None):
        self.name = name
        self.default = default

    def to_xml(self):
        raise NotImplementedError()


class StringField(BaseField):
    converter = six.text_type

    def to_xml(self, value):
        value = self.converter(value)
        tpl = '<{name}>![CDATA[{value}]]</{name}>'
        return tpl.format(name=self.name, value=value)


class IntegerField(BaseField):
    converter = int

    def to_xml(self, value):
        value = self.converter(value)
        tpl = '<{name}>{value}</{name}>'
        return tpl.format(name=self.name, value=value)


class FloatField(BaseField):
    converter = float

    def to_xml(self, value):
        value = self.converter(value)
        tpl = '<{name}>{value}</{name}>'
        return tpl.format(name=self.name, value=value)


class ImageField(StringField):

    def to_xml(self, value):
        value = self.converter(value)
        tpl = '<Image>\n<MediaId>![CDATA[{value}]]</MediaId>\n</Image>'
        return tpl.format(value=value)


class VoiceField(StringField):

    def to_xml(self, value):
        value = self.converter(value)
        tpl = '<Voice>\n<MediaId>![CDATA[{value}]]</MediaId>\n</Voice>'
        return tpl.format(value=value)


class VideoField(StringField):

    def to_xml(self, media_id, title=None, description=None):
        media_id = self.converter(media_id)
        if title:
            title = self.converter(title)
        if description:
            description = self.converter(description)
        tpl = """<Video>
        <MediaId>![CDATA[{media_id}]]</MediaId>
        <Title>![CDATA[{title}]]</Title>
        <Description>![CDATA[{description}]]</Description>
        </Video>
        """
        return tpl.format(
            media_id=media_id,
            title=title,
            description=description
        )


class MusicField(StringField):

    def to_xml(self, thumb_media_id, title=None, description=None,
               music_url=None, hq_music_url=None):
        thumb_media_id = self.converter(thumb_media_id)
        if title:
            title = self.converter(title)
        if description:
            description = self.converter(description)
        if music_url:
            music_url = self.converter(music_url)
        if hq_music_url:
            hq_music_url = self.converter(hq_music_url)
        tpl = """<Music>
        <ThumbMediaId>![CDATA[{thumb_media_id}]]</ThumbMediaId>
        <Title>![CDATA[{title}]]</Title>
        <Description>![CDATA[{description}]]</Description>
        <MusicUrl>![CDATA[{music_url}]]</MusicUrl>
        <HQMusicUrl>![CDATA[{hq_music_url}]]</HQMusicUrl>
        </Music>"""
        return tpl.format(
            thumb_media_id=thumb_media_id,
            title=title,
            description=description,
            music_url=music_url,
            hq_music_url=hq_music_url
        )


class ArticleField(StringField):

    def to_xml(self, articles):
        article_count = len(articles)
        items = []
        for article in articles:
            title = self.converter(article.get('title', ''))
            description = self.converter(article.get('description', ''))
            image = self.converter(article.get('image', ''))
            url = self.converter(article.get('url', ''))
            item_tpl = """<item>
            <Title>![CDATA[{title}]]</Title>
            <Description>![CDATA[{description}]]</Description>
            <PicUrl>![CDATA[{image}]]</PicUrl>
            <Url>![CDATA[{url}]]</Url>
            </item>
            """
            item = item_tpl.format(
                title=title,
                description=description,
                image=image,
                url=url
            )
            items.append(item)
        items_str = '\n'.join(items)
        tpl = """<ArticleCount>{article_count}</ArticleCount>
        <Articles>{items}</Articles>
        """
        return tpl.format(
            article_count=article_count,
            items=items_str
        )
{'fraction_non_alphanumeric': 0.13651347682761295, 'fraction_numerical': 0.0, 'mean_word_length': 2.4501672240802677, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 13}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6771059', 'n_tokens_mistral': 1637, 'n_tokens_neox': 1551, 'n_words': 269}