<commit_before>import sys
import traceback
from difflib import SequenceMatcher

from django.conf import settings
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import get_object_or_404, render_to_response
from django.template import RequestContext
from djblets.util import cache_memoize

from reviewboard.diffviewer.forms import UploadDiffForm
from reviewboard.diffviewer.models import DiffSet, FileDiff
import reviewboard.diffviewer.diffutils as diffutils
import reviewboard.scmtools as scmtools


class UserVisibleError(Exception):
    pass


def get_diff_files(diffset):
    def get_original_file(file, revision):
        """Get a file either from the cache or the SCM.  SCM exceptions are
           passed back to the caller."""
        try:
            return cache_memoize(file,
                lambda: scmtools.get_tool().get_file(file, revision))
        except Exception, e:
            raise UserVisibleError(str(e))

    def get_chunks(filediff):
        def diff_line(linenum, oldline, newline):
            region = diffutils.get_line_changed_regions(oldline, newline)
            return [linenum, oldline or "", region[0], newline or "", region[1]]

        def new_chunk(lines, numlines, tag, collapsable=False):<commit_msg>Be a lot smarter about when to show inter-line diffs<commit_after>import sys
import traceback
from difflib import SequenceMatcher

from django.conf import settings
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import get_object_or_404, render_to_response
from django.template import RequestContext
from djblets.util import cache_memoize

from reviewboard.diffviewer.forms import UploadDiffForm
from reviewboard.diffviewer.models import DiffSet, FileDiff
import reviewboard.diffviewer.diffutils as diffutils
import reviewboard.scmtools as scmtools


class UserVisibleError(Exception):
    pass


def get_diff_files(diffset):
    def get_original_file(file, revision):
        """Get a file either from the cache or the SCM.  SCM exceptions are
           passed back to the caller."""
        try:
            return cache_memoize(file,
                lambda: scmtools.get_tool().get_file(file, revision))
        except Exception, e:
            raise UserVisibleError(str(e))

    def get_chunks(filediff):
        def diff_line(linenum, oldline, newline):
            if not oldline or not newline:
                return [linenum, oldline or '', [], newline or '', []]

            region = diffutils.get_line_changed_regions(oldline, newline)
            print region

            # We only want inter-line diffs for small changes.  This somewhat
            # dense block of code computes the ratio of changed/not-changed.
            # If that ratio is more than 25% of the line, we don't show changes
            # inline.
            changed = (region[0] and sum(y - x for x, y in region[0]) or 0) + \
                      (region[1] and sum(y - x for x, y in region[1]) or 0)

            oldline = oldline or ''
            newline = newline or ''
            if float(changed) / (len(oldline) + len(newline)) > 0.25:
                region = ([], [])

            return [linenum, oldline, region[0], newline, region[1]]

        def new_chunk(lines, numlines, tag, collapsable=False):
{'fraction_non_alphanumeric': 0.07589960689446629, 'fraction_numerical': 0.006350166313879649, 'mean_word_length': 3.0096969696969698, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7277204', 'n_tokens_mistral': 949, 'n_tokens_neox': 910, 'n_words': 301}