<commit_before># Copyright 2008 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Diff rendering in HTML for Rietveld."""

# Python imports
import re
import cgi
import difflib
import logging
import urlparse

# AppEngine imports
from google.appengine.api import urlfetch
from google.appengine.api import users
from google.appengine.ext import db

# Django imports
from django.template import loader

# Local imports
import models
import patching
import intra_region_diff


class FetchError(Exception):
  """Exception raised by FetchBase() when a URL problem occurs."""


# NOTE: this function is duplicated in upload.py, keep them in sync.
def SplitPatch(data):
  """Splits a patch into separate pieces for each file.

  Args:
    data: A string containing the output of svn diff.

  Returns:
    A list of 2-tuple (filename, text) where text is the svn diff output
      pertaining to filename.
  """
  patches = []
  filename = None
  diff = []
  for line in data.splitlines(True):
    new_filename = None
    if line.startswith('Index:'):
      unused, new_filename = line.split(':', 1)
      new_filename = new_filename.strip()
    elif line.startswith('Property changes on:'):
      unused, temp_filename = line.split(':', 1)
      # When a file is modified, paths use '/' between directories, however
      # when a property is modified '\' is used on Windows.  Make them the same
      # otherwise the file shows up twice.
      temp_filename = temp_filename.strip().replace('\\', '/')
      if temp_filename != filename:
        # File has property changes but no modifications, create a new diff.
        new_filename = temp_filename
    if new_filename:
      if filename and diff:
        patches.append((filename, ''.join(diff)))
      filename = new_filename
      diff = [line]
      continue
    if diff is not None:
      diff.append(line)
  if filename and diff:
    patches.append((filename, ''.join(diff)))
  return patches


def ParsePatchSet(patchset):
  """Patch a patch set into individual patches.

  Args:
    patchset: a models.PatchSet instance.

  Returns:
    A list of models.Patch instances.
  """
  patches = []
  for filename, text in SplitPatch(patchset.data):
    patches.append(models.Patch(patchset=patchset, text=ToText(text),
                                filename=filename, parent=patchset))
  return patches


def FetchBase(base, patch):
  """Fetch the content of the file to which the file is relative.

  Args:
    base: the base property of the Issue to which the Patch belongs.
    patch: a models.Patch instance.

  Returns:
    A models.Content instance.

  Raises:
    FetchError: For any kind of problem fetching the content.
  """
  filename, lines = patch.filename, patch.lines
  rev = patching.ParseRevision(lines)
  if rev is not None:
    if rev == 0:
      # rev=0 means it's a new file.
      return models.Content(text=db.Text(u''), parent=patch)
  try:
    base = db.Link(base)
  except db.BadValueError:
    msg = 'Invalid base URL: %s' % base
    logging.error(msg)
    raise FetchError(msg)
  url = _MakeUrl(base, filename, rev)
  logging.info('Fetching %s', url)
  try:
    result = urlfetch.fetch(url)
  except Exception, err:
    msg = 'Error fetching %s: %s: %s' % (url, err.__class__.__name__, err)
    logging.warn('FetchBase: %s', msg)
    raise FetchError(msg)
  if result.status_code != 200:
    msg = 'Error fetching %s: HTTP status %s' % (url, result.status_code)
    logging.warn('FetchBase: %s', msg)
    raise FetchError(msg)
  return models.Content(text=ToText(result.content), parent=patch,
                        is_complete=True)


def _MakeUrl(base, filename, rev):
  """Helper for FetchBase() to construct the URL to fetch.

  Args:
    base: The base property of the Issue to which the Patch belongs.
    filename: The filename property of the Patch instance.
    rev: Revision number, or None for head revision.
<commit_msg>Issue  3070: Removed unused is_complete keyword. <commit_after># Copyright 2008 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Diff rendering in HTML for Rietveld."""

# Python imports
import re
import cgi
import difflib
import logging
import urlparse

# AppEngine imports
from google.appengine.api import urlfetch
from google.appengine.api import users
from google.appengine.ext import db

# Django imports
from django.template import loader

# Local imports
import models
import patching
import intra_region_diff


class FetchError(Exception):
  """Exception raised by FetchBase() when a URL problem occurs."""


# NOTE: this function is duplicated in upload.py, keep them in sync.
def SplitPatch(data):
  """Splits a patch into separate pieces for each file.

  Args:
    data: A string containing the output of svn diff.

  Returns:
    A list of 2-tuple (filename, text) where text is the svn diff output
      pertaining to filename.
  """
  patches = []
  filename = None
  diff = []
  for line in data.splitlines(True):
    new_filename = None
    if line.startswith('Index:'):
      unused, new_filename = line.split(':', 1)
      new_filename = new_filename.strip()
    elif line.startswith('Property changes on:'):
      unused, temp_filename = line.split(':', 1)
      # When a file is modified, paths use '/' between directories, however
      # when a property is modified '\' is used on Windows.  Make them the same
      # otherwise the file shows up twice.
      temp_filename = temp_filename.strip().replace('\\', '/')
      if temp_filename != filename:
        # File has property changes but no modifications, create a new diff.
        new_filename = temp_filename
    if new_filename:
      if filename and diff:
        patches.append((filename, ''.join(diff)))
      filename = new_filename
      diff = [line]
      continue
    if diff is not None:
      diff.append(line)
  if filename and diff:
    patches.append((filename, ''.join(diff)))
  return patches


def ParsePatchSet(patchset):
  """Patch a patch set into individual patches.

  Args:
    patchset: a models.PatchSet instance.

  Returns:
    A list of models.Patch instances.
  """
  patches = []
  for filename, text in SplitPatch(patchset.data):
    patches.append(models.Patch(patchset=patchset, text=ToText(text),
                                filename=filename, parent=patchset))
  return patches


def FetchBase(base, patch):
  """Fetch the content of the file to which the file is relative.

  Args:
    base: the base property of the Issue to which the Patch belongs.
    patch: a models.Patch instance.

  Returns:
    A models.Content instance.

  Raises:
    FetchError: For any kind of problem fetching the content.
  """
  filename, lines = patch.filename, patch.lines
  rev = patching.ParseRevision(lines)
  if rev is not None:
    if rev == 0:
      # rev=0 means it's a new file.
      return models.Content(text=db.Text(u''), parent=patch)
  try:
    base = db.Link(base)
  except db.BadValueError:
    msg = 'Invalid base URL: %s' % base
    logging.error(msg)
    raise FetchError(msg)
  url = _MakeUrl(base, filename, rev)
  logging.info('Fetching %s', url)
  try:
    result = urlfetch.fetch(url)
  except Exception, err:
    msg = 'Error fetching %s: %s: %s' % (url, err.__class__.__name__, err)
    logging.warn('FetchBase: %s', msg)
    raise FetchError(msg)
  if result.status_code != 200:
    msg = 'Error fetching %s: HTTP status %s' % (url, result.status_code)
    logging.warn('FetchBase: %s', msg)
    raise FetchError(msg)
  return models.Content(text=ToText(result.content), parent=patch)


def _MakeUrl(base, filename, rev):
  """Helper for FetchBase() to construct the URL to fetch.

  Args:
    base: The base property of the Issue to which the Patch belongs.
    filename: The filename property of the Patch instance.
    rev: Revision number, or None for head revision.

{'fraction_non_alphanumeric': 0.09483929379809869, 'fraction_numerical': 0.004074241738343142, 'mean_word_length': 3.306530214424951, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4360557', 'n_tokens_mistral': 2597, 'n_tokens_neox': 2530, 'n_words': 993}