<commit_before>from __future__ import unicode_literals

import logging

from django.utils import six
from django.utils.datastructures import SortedDict
from django.utils.html import escape, strip_tags
from django.utils.safestring import mark_safe

from reviewboard.diffviewer.diffutils import get_line_changed_regions
from reviewboard.diffviewer.myersdiff import MyersDiffer
from reviewboard.diffviewer.templatetags.difftags import highlightregion
from reviewboard.reviews.markdown_utils import (is_rich_text_default_for_user,
                                                iter_markdown_lines,
                                                normalize_text_for_edit,
                                                render_markdown)


_all_fields = {}
_fieldsets = SortedDict()
_populated = False


class BaseReviewRequestFieldSet(object):
    """Base class for sets of review request fields.

    A fieldset stores a list of fields that are rendered on the review
    request page. They may contain default fields, and new fields can be
    added or removed.

    Review Board provides three main fieldsets: "main", "info", and
    "reviewers". Others can be added by subclassing and registering
    through ``register_review_request_fieldset``.
    """
    fieldset_id = None
    label = None
    show_required = False
    field_classes = None
    tag_name = None

    def __init__(self, review_request_details):
        self.review_request_details = review_request_details

    @classmethod
    def is_empty(cls):
        """Returns whether the fieldset is empty.

        A fieldset is empty if there are no field classes registered.
        An empty fieldset will not be displayed on the page.
        """
        return not cls.field_classes

    @classmethod
    def add_field(cls, field_cls):
        """Adds a field class to this fieldset.

        The field will be rendered inside this fieldset on the page.

        A given field class can only be in one fieldset. Its ``field_id``
        must be unique.
        """
        _register_field(field_cls)
        cls.field_classes.append(field_cls)

    @classmethod
    def remove_field(cls, field_cls):
        """Removes a field class from this fieldset.

        The field class must have been previously added to this fieldset.
        """
        field_id = field_cls.field_id

        try:
            cls.field_classes.remove(field_cls)
            del _all_fields[field_id]
        except KeyError:
            logging.error('Failed to unregister unknown review request '
                          'field "%s"',
                          field_id)
            raise KeyError('"%s" is not a registered review request field'
                           % field_id)


class BaseReviewRequestField(object):
    """Base class for a field on a review request.

    A field is responsible for displaying data from a review request,
    handling any editing requirements if necessary, recording changes
    in the ChangeDescription, and rendering those changes.

    Each field must have its own unique ``field_id``. This ID will be used
    when looking up or storing the field's value.

    It is recommended that fields provided by extensions prefix their
    field ID with some sort of identifier for the extension or the vendor.

    Creating a new field requires subclassing BaseReviewRequestField and
    overriding any fields or functions necessary. Its class must then be
    added to a fieldset.

    A field will be instantiated with either a ReviewRequest or a
    ReviewRequestDraft, depending on what is being edited. This is stored
    in ``review_request_details``. Functions should optimistically fetch
    values from that, if possible. They can call ``get_review_request()``
    on ``review_request_details`` to fetch the actual ReviewRequest.

    If the function takes a ``review_request_details`` parameter, it must
    use that instead.
    """
    field_id = None
    label = None
    is_editable = False
    is_required = False
    default_css_classes = set()
    change_entry_renders_inline = True
    model = None

    can_record_change_entry = property(lambda self: self.is_editable)

    def __init__(self, review_request_details, request=None):
        self.review_request_details = review_request_details
        self.request = request

    @property
    def value(self):
        """Returns the value loaded from the database.

        This will fetch the value with the associated ReviewRequest or
        ReviewRequestDraft, and then cache it for future lookups.
        """
        if not hasattr(self, '_value'):
            self._value = self.load_value(self.review_request_details)

        return self._value

    def has_value_changed(self, old_value, new_value):
        """Returns whether the value has changed.

        By default, it performs an inequality check on the values. This
        can be overridden to perform more specialized checks.
        """
        return old_value != new_value

    def record_change_entry(self, changedesc, old_value, new_value):
        """Records information on the changed values in a ChangeDescription.

        By default, the values are stored as-is along with the field ID.
        This can be overridden to perform more specialized storage.
        """
        changedesc.record_field_change(self.field_id, old_value, new_value)

    def serialize_change_entry(self, changedesc):
        """Serializes the change entry for public consumption.

        This will output a version of the change entry for use in the API.
        It can be the same content stored in the ChangeDescription, but
        does not need to be.
        """
        field_info = changedesc.fields_changed[self.field_id]

        if self.model:
            pks = [
                value[2]
                for key in ('new', 'old', 'added', 'removed')
                for value in field_info[key]
            ]
            pk_to_objects = dict([
                (obj.pk, obj)
                for obj in self.model.objects.filter(pk__in=pks)
            ])

            return dict([
                (key, pk_to_objects[field_info[key][2]])
                for key in ('new', 'old', 'added', 'removed')
                if key in field_info
            ])
        else:
            return dict([
                (key, field_info[key][0])
                for key in ('new', 'old', 'added', 'removed')
                if key in field_info
            ])

    def _serialize_change_entry_value(self, value):
        if self.model:
            return self.model.objects.get(pk=value[2])
        else:
            return value[0]

    def get_change_entry_sections_html(self, info):
        """Returns sections of change entries with titles and rendered HTML.

        By default, this just returns a single section for the field, with
        the field's title and rendered change HTML.

        Subclasses can override this to provide more information.
        """
        return [{
            'title': self.label,
            'rendered_html': mark_safe(self.render_change_entry_html(info)),
        }]

    def render_change_entry_html(self, info):
        """Renders a change entry to HTML.

        By default, this returns a simple "changed from X to Y" using the old
        and new values. This can be overridden to generate more specialized
        output.

        This function is expected to return safe, valid HTML. Any values
        coming from a field or any other form of user input must be
        properly escaped.

        Subclasses can override ``render_change_entry_value_html`` to
        change how the value itself will be rendered in the string.
        """
        old_value = ''
        new_value = ''

        if 'old' in info:
            old_value = info['old'][0]

        if 'new' in info:
            new_value = info['new'][0]

        s = ['<table class="changed">']

        if old_value:
            s.append(self.render_change_entry_removed_value_html(
                info, old_value))

        if new_value:
            s.append(self.render_change_entry_added_value_html(
                info, new_value))

        s.append('</table>')

        return ''.join(s)

    def render_change_entry_added_value_html(self, info, value):
        value_html = self.render_change_entry_value_html(info, value)

        if value_html:
            return ('<tr class="new-value"><th class="marker">+</th>'
                    '<td class="value">%s</td></tr>' % value_html)
        else:
            return ''

    def render_change_entry_removed_value_html(self, info, value):
        value_html = self.render_change_entry_value_html(info, value)

        if value_html:
            return ('<tr class="old-value"><th class="marker">-</th>'
                    '<td class="value">%s</td></tr>' % value_html)
        else:
            return ''

    def render_change_entry_value_html(self, info, value):
        """Renders the value for a change description string to HTML.

        By default, this just converts the value to text and escapes it.
        This can be overridden to customize how the value is displayed.
        """
        return escape(six.text_type(value or ''))

    def load_value(self, review_request_details):
        """Loads a value from the review request or draft.

        By default, this loads the value as-is from the extra_data field.
        This can be overridden if you need to deserialize the value in some
        way.

        This must use ``review_request_details`` instead of
        ``self.review_request_details``.
        """
        return review_request_details.extra_data.get(self.field_id)

    def save_value(self, value):
        """Saves the value in the review request or draft.

        By default, this saves the value as-is in the extra_data field.
        This can be overridden if you need to serialize the value in some
        way.
        """
        self.review_request_details.extra_data[self.field_id] = value

    def render_value(self, value):
        """Renders the value in the field.

        By default, this converts to text and escapes it. This can be
        overridden if you need to render it in a more specific way.

        This must use ``value`` instead of ``self.value``.
        """
        return escape(six.text_type(value or ''))

    def should_render(self, value):
        """Returns whether the field should be rendered.

        By default, the field is always rendered, but this can be overridden
        if you only want to show under certain conditions (such as if it has
        a value).

        This must use ``value`` instead of ``self.value``.
        """
        return True

    def get_css_classes(self):
        """Returns the list of CSS classes to apply to the element.

        By default, this will include the contents of ``default_css_classes``,
        and ``required`` if it's a required field.

        This can be overridden to provide additional CSS classes, if they're
        not appropraite for ``default_css_classes``.
        """
        css_classes = set(self.default_css_classes)

        if self.is_required:
            css_classes.add('required')

        return css_classes

    def get_data_attributes(self):
        """Returns any data attributes to include in the element.

        By default, this returns nothing.
        """
        return {}

    def as_html(self):
        """Returns the field rendered as HTML.

        By default, this just calls ``render_value`` with the value
        from the database.
        """
        return self.render_value(self.value)


class BaseEditableField(BaseReviewRequestField):
    """Base class for an editable field.

    This simply marks the field as editable.
    """
    default_css_classes = ['editable']
    is_editable = True


class BaseCommaEditableField(BaseEditableField):
    """Base class for an editable comma-separated list of values.

    This is used for dealing with lists of items that appear
    comma-separated in the UI. It works with stored lists of content
    on the review request or draft, and on the ChangeDescription.

    Subclasses can override this to provide specialized rendering
    on a per-item-basis. That's useful for showing links to items,
    for example.
    """
    default_css_classes = ['editable', 'comma-editable']
    order_matters = False

    one_line_per_change_entry = True

    def has_value_changed(self, old_value, new_value):
        """Returns whether two values have changed.

        If ``order_matters`` is set to ``True``, this will do a strict
        list comparison. Otherwise, it will compare the items in both
        lists without caring about the ordering.
        """
        if self.order_matters:
            return old_value != new_value
        else:
            return set(old_value or []) != set(new_value or [])

    def serialize_change_entry(self, changedesc):
        """Serializes the change entry for public consumption.

        This will output a version of the change entry for use in the API.
        It can be the same content stored in the ChangeDescription, but
        does not need to be.
        """
        field_info = changedesc.fields_changed[self.field_id]

        if self.model:
            pks = [
                value[2]
                for key in ('new', 'old', 'added', 'removed')
                for value in field_info[key]
            ]
            pk_to_objects = dict([
                (obj.pk, obj)
                for obj in self.model.objects.filter(pk__in=pks)
            ])

            return dict([
                (key, [
                    pk_to_objects[value[2]]
                    for value in field_info[key]
                ])
                for key in ('new', 'old', 'added', 'removed')
                if key in field_info
            ])
        else:
            return dict([
                (key, [
                    value[0]
                    for value in field_info[key]
                ])
                for key in ('new', 'old', 'added', 'removed')
                if key in field_info
            ])

    def render_value(self, values):
        """Renders the list of items.

        This will call out to ``render_item`` for every item. The list
        of rendered items will be separated by a comma and a space.
        """
        return ', '.join([
            self.render_item(value)
            for value in values
        ])

    def render_item(self, item):
        """Renders an item from the list.

        By default, this will convert the item to text and then escape it.
        """
        return escape(six.text_type(item or ''))
<commit_msg>Refactor change description field serialization<commit_after>from __future__ import unicode_literals

import logging

from django.utils import six
from django.utils.datastructures import SortedDict
from django.utils.html import escape, strip_tags
from django.utils.safestring import mark_safe

from reviewboard.diffviewer.diffutils import get_line_changed_regions
from reviewboard.diffviewer.myersdiff import MyersDiffer
from reviewboard.diffviewer.templatetags.difftags import highlightregion
from reviewboard.reviews.markdown_utils import (is_rich_text_default_for_user,
                                                iter_markdown_lines,
                                                normalize_text_for_edit,
                                                render_markdown)


_all_fields = {}
_fieldsets = SortedDict()
_populated = False


class BaseReviewRequestFieldSet(object):
    """Base class for sets of review request fields.

    A fieldset stores a list of fields that are rendered on the review
    request page. They may contain default fields, and new fields can be
    added or removed.

    Review Board provides three main fieldsets: "main", "info", and
    "reviewers". Others can be added by subclassing and registering
    through ``register_review_request_fieldset``.
    """
    fieldset_id = None
    label = None
    show_required = False
    field_classes = None
    tag_name = None

    def __init__(self, review_request_details):
        self.review_request_details = review_request_details

    @classmethod
    def is_empty(cls):
        """Returns whether the fieldset is empty.

        A fieldset is empty if there are no field classes registered.
        An empty fieldset will not be displayed on the page.
        """
        return not cls.field_classes

    @classmethod
    def add_field(cls, field_cls):
        """Adds a field class to this fieldset.

        The field will be rendered inside this fieldset on the page.

        A given field class can only be in one fieldset. Its ``field_id``
        must be unique.
        """
        _register_field(field_cls)
        cls.field_classes.append(field_cls)

    @classmethod
    def remove_field(cls, field_cls):
        """Removes a field class from this fieldset.

        The field class must have been previously added to this fieldset.
        """
        field_id = field_cls.field_id

        try:
            cls.field_classes.remove(field_cls)
            del _all_fields[field_id]
        except KeyError:
            logging.error('Failed to unregister unknown review request '
                          'field "%s"',
                          field_id)
            raise KeyError('"%s" is not a registered review request field'
                           % field_id)


class BaseReviewRequestField(object):
    """Base class for a field on a review request.

    A field is responsible for displaying data from a review request,
    handling any editing requirements if necessary, recording changes
    in the ChangeDescription, and rendering those changes.

    Each field must have its own unique ``field_id``. This ID will be used
    when looking up or storing the field's value.

    It is recommended that fields provided by extensions prefix their
    field ID with some sort of identifier for the extension or the vendor.

    Creating a new field requires subclassing BaseReviewRequestField and
    overriding any fields or functions necessary. Its class must then be
    added to a fieldset.

    A field will be instantiated with either a ReviewRequest or a
    ReviewRequestDraft, depending on what is being edited. This is stored
    in ``review_request_details``. Functions should optimistically fetch
    values from that, if possible. They can call ``get_review_request()``
    on ``review_request_details`` to fetch the actual ReviewRequest.

    If the function takes a ``review_request_details`` parameter, it must
    use that instead.
    """
    field_id = None
    label = None
    is_editable = False
    is_required = False
    default_css_classes = set()
    change_entry_renders_inline = True
    model = None

    can_record_change_entry = property(lambda self: self.is_editable)

    def __init__(self, review_request_details, request=None):
        self.review_request_details = review_request_details
        self.request = request

    @property
    def value(self):
        """Returns the value loaded from the database.

        This will fetch the value with the associated ReviewRequest or
        ReviewRequestDraft, and then cache it for future lookups.
        """
        if not hasattr(self, '_value'):
            self._value = self.load_value(self.review_request_details)

        return self._value

    def has_value_changed(self, old_value, new_value):
        """Returns whether the value has changed.

        By default, it performs an inequality check on the values. This
        can be overridden to perform more specialized checks.
        """
        return old_value != new_value

    def record_change_entry(self, changedesc, old_value, new_value):
        """Records information on the changed values in a ChangeDescription.

        By default, the values are stored as-is along with the field ID.
        This can be overridden to perform more specialized storage.
        """
        changedesc.record_field_change(self.field_id, old_value, new_value)

    def serialize_change_entry(self, changedesc):
        """Serialize a change entry for public consumption.

        This will output a version of the change entry for use in the API.
        It can be the same content stored in the
        :py:class:`~reviewboard.changedescs.models.ChangeDescription`, but
        does not need to be.

        Args:
            changedesc (reviewboard.changedescs.models.ChangeDescription):
                The change description whose field is to be serialized.

        Returns:
            dict:
            An appropriate serialization for the field.
        """
        field_info = changedesc.fields_changed[self.field_id]

        if self.model:
            return self.serialize_change_entry_for_model_list(field_info)
        else:
            return self.serialize_change_entry_for_singleton(field_info)

    def serialize_change_entry_for_model_list(self, field_info):
        """Return the change entry for a list of models.

        Args:
            field_info (dict):
                A dictionary describing how the field has changed. This is
                guaranteed to have ``new`` and ``old`` keys, but may also
                contain ``added`` and ``removed`` keys as well.

        Returns:
            dict:
            A mapping of each key present in ``field_info`` to its list of
            model instances.
        """
        pks = [
            value[2]
            for key in ('new', 'old', 'added', 'removed')
            if key in field_info
            for value in field_info[key]
        ]
        pk_to_objects = dict([
            (obj.pk, obj)
            for obj in self.model.objects.filter(pk__in=pks)
        ])

        return dict([
            (key, [
                pk_to_objects[value[2]]
                for value in field_info[key]
            ])
            for key in ('new', 'old', 'added', 'removed')
            if key in field_info
        ])

    def serialize_change_entry_for_singleton(self, field_info):
        """Return the change entry for a singleton.

        Singleton fields (e.g., summaries) are stored in
        :py:class:`~reviewboard.changedescs.models.ChangeDescription`s as
        a list with a single element.

        Args:
            field_info (dict):
                A dictionary describing how the field has changed. This is
                guaranteed to have ``new`` and ``old`` keys, but may also
                contain ``added`` and ``removed`` keys as well.

        Returns:
            dict:
            A mapping of each key in ``field_info`` to a single value.
        """
        return dict([
            (key, field_info[key][0])
            for key in ('new', 'old', 'added', 'removed')
            if key in field_info
        ])

    def serialize_change_entry_for_list(self, field_info):
        """Return the change entry for a list of plain data.

        Args:
            field_info (dict):
                A dictionary describing how the field has changed. This is
                guaranteed to have ``new`` and ``old`` keys, but may also
                contain ``added`` and ``removed`` keys as well.

        Returns:
            dict:
            A mapping of each key in ``field_info`` to a list of values.
        """
        return dict([
            (key, [
                value[0]
                for value in field_info[key]
            ])
            for key in ('new', 'old', 'added', 'removed')
            if key in field_info
        ])

    def get_change_entry_sections_html(self, info):
        """Returns sections of change entries with titles and rendered HTML.

        By default, this just returns a single section for the field, with
        the field's title and rendered change HTML.

        Subclasses can override this to provide more information.
        """
        return [{
            'title': self.label,
            'rendered_html': mark_safe(self.render_change_entry_html(info)),
        }]

    def render_change_entry_html(self, info):
        """Renders a change entry to HTML.

        By default, this returns a simple "changed from X to Y" using the old
        and new values. This can be overridden to generate more specialized
        output.

        This function is expected to return safe, valid HTML. Any values
        coming from a field or any other form of user input must be
        properly escaped.

        Subclasses can override ``render_change_entry_value_html`` to
        change how the value itself will be rendered in the string.
        """
        old_value = ''
        new_value = ''

        if 'old' in info:
            old_value = info['old'][0]

        if 'new' in info:
            new_value = info['new'][0]

        s = ['<table class="changed">']

        if old_value:
            s.append(self.render_change_entry_removed_value_html(
                info, old_value))

        if new_value:
            s.append(self.render_change_entry_added_value_html(
                info, new_value))

        s.append('</table>')

        return ''.join(s)

    def render_change_entry_added_value_html(self, info, value):
        value_html = self.render_change_entry_value_html(info, value)

        if value_html:
            return ('<tr class="new-value"><th class="marker">+</th>'
                    '<td class="value">%s</td></tr>' % value_html)
        else:
            return ''

    def render_change_entry_removed_value_html(self, info, value):
        value_html = self.render_change_entry_value_html(info, value)

        if value_html:
            return ('<tr class="old-value"><th class="marker">-</th>'
                    '<td class="value">%s</td></tr>' % value_html)
        else:
            return ''

    def render_change_entry_value_html(self, info, value):
        """Renders the value for a change description string to HTML.

        By default, this just converts the value to text and escapes it.
        This can be overridden to customize how the value is displayed.
        """
        return escape(six.text_type(value or ''))

    def load_value(self, review_request_details):
        """Loads a value from the review request or draft.

        By default, this loads the value as-is from the extra_data field.
        This can be overridden if you need to deserialize the value in some
        way.

        This must use ``review_request_details`` instead of
        ``self.review_request_details``.
        """
        return review_request_details.extra_data.get(self.field_id)

    def save_value(self, value):
        """Saves the value in the review request or draft.

        By default, this saves the value as-is in the extra_data field.
        This can be overridden if you need to serialize the value in some
        way.
        """
        self.review_request_details.extra_data[self.field_id] = value

    def render_value(self, value):
        """Renders the value in the field.

        By default, this converts to text and escapes it. This can be
        overridden if you need to render it in a more specific way.

        This must use ``value`` instead of ``self.value``.
        """
        return escape(six.text_type(value or ''))

    def should_render(self, value):
        """Returns whether the field should be rendered.

        By default, the field is always rendered, but this can be overridden
        if you only want to show under certain conditions (such as if it has
        a value).

        This must use ``value`` instead of ``self.value``.
        """
        return True

    def get_css_classes(self):
        """Returns the list of CSS classes to apply to the element.

        By default, this will include the contents of ``default_css_classes``,
        and ``required`` if it's a required field.

        This can be overridden to provide additional CSS classes, if they're
        not appropraite for ``default_css_classes``.
        """
        css_classes = set(self.default_css_classes)

        if self.is_required:
            css_classes.add('required')

        return css_classes

    def get_data_attributes(self):
        """Returns any data attributes to include in the element.

        By default, this returns nothing.
        """
        return {}

    def as_html(self):
        """Returns the field rendered as HTML.

        By default, this just calls ``render_value`` with the value
        from the database.
        """
        return self.render_value(self.value)


class BaseEditableField(BaseReviewRequestField):
    """Base class for an editable field.

    This simply marks the field as editable.
    """
    default_css_classes = ['editable']
    is_editable = True


class BaseCommaEditableField(BaseEditableField):
    """Base class for an editable comma-separated list of values.

    This is used for dealing with lists of items that appear
    comma-separated in the UI. It works with stored lists of content
    on the review request or draft, and on the ChangeDescription.

    Subclasses can override this to provide specialized rendering
    on a per-item-basis. That's useful for showing links to items,
    for example.
    """
    default_css_classes = ['editable', 'comma-editable']
    order_matters = False

    one_line_per_change_entry = True

    def has_value_changed(self, old_value, new_value):
        """Returns whether two values have changed.

        If ``order_matters`` is set to ``True``, this will do a strict
        list comparison. Otherwise, it will compare the items in both
        lists without caring about the ordering.
        """
        if self.order_matters:
            return old_value != new_value
        else:
            return set(old_value or []) != set(new_value or [])

    def serialize_change_entry(self, changedesc):
        """Serialize a change entry for public consumption.

        This will output a version of the change entry for use in the API.
        It can be the same content stored in the
        :py:class:`~reviewboard.changedescs.models.ChangeDescription`, but
        does not need to be.

        Args:
            changedesc (reviewboard.changedescs.models.ChangeDescription):
                The change description whose field is to be serialized.

        Returns:
            dict:
            An appropriate serialization for the field.
        """
        field_info = changedesc.fields_changed[self.field_id]

        if self.model:
            return self.serialize_change_entry_for_model_list(field_info)
        else:
            return self.serialize_change_entry_for_list(field_info)

    def render_value(self, values):
        """Renders the list of items.

        This will call out to ``render_item`` for every item. The list
        of rendered items will be separated by a comma and a space.
        """
        return ', '.join([
            self.render_item(value)
            for value in values
        ])

    def render_item(self, item):
        """Renders an item from the list.

        By default, this will convert the item to text and then escape it.
        """
        return escape(six.text_type(item or ''))

{'fraction_non_alphanumeric': 0.07300856667945275, 'fraction_numerical': 0.000511443549418233, 'mean_word_length': 2.224260537977945, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2109107', 'n_tokens_mistral': 8267, 'n_tokens_neox': 7982, 'n_words': 3063}