<commit_before>#!/usr/bin/python2
"""
Copyright 2013 Google Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

"""A tool to show the differnce in URLs.

A tool for showing the differnces between URLs.  This is inspired by the unix
utility diff.
"""

import argparse
import copy
import logging
import sys

class Error(Exception):
  """Base exception class."""
  pass

class ParamDiffTypeError(Error):
  """Raised when an incorrect diff type used."""
  pass

class HostnameParseError(Error):
  """Raised when unable to parse hostname from URL."""
  pass

# TODO(macpd): investigate making this a namedtuple
class ParamDiffEntry(object):
  """Represents difference of 2 URL params with same name."""
  LEFT_ONLY = 1
  RIGHT_ONLY = 2
  BOTH_DIFFER = 3
  LEFT_VALUE_FORMAT = '{0}\n< {1}'
  RIGHT_VALUE_FORMAT = '{0}\n> {1}'

  def __init__(self, name, left_value, right_value, diff_type):
    self._name = name
    self._left_val = left_value
    self._right_val = right_value
    try:
      if self._valid_diff_type(diff_type):
        self._type = diff_type
    except ParamDiffTypeError:
      logging.error("Incorrect diff type: %s", diff_type)
      self._type = self.BOTH_DIFFER

  def _valid_diff_type(self, diff_type):
    if (diff_type != self.LEFT_ONLY and diff_type != self.RIGHT_ONLY and
        diff_type != self.BOTH_DIFFER):
      raise ParamDiffTypeError('%s is not a valid diff type. ', diff_type)
    return True

  @property
  def name(self):
    return self._name

  def __str__(self):
    ret = self._name
    if self._type == self.LEFT_ONLY or self._type == self.BOTH_DIFFER:
      ret = self.LEFT_VALUE_FORMAT.format(ret, self._left_val)
    if self._type == self.RIGHT_ONLY or self._type == self.BOTH_DIFFER:
      ret = self.RIGHT_VALUE_FORMAT.format(ret, self._right_val)
    return ret


class UrlDiffer(object):
  """Object to diff URLs.

  Diffs URLs upon intialization."""

  PATH_DELIM = '?'
  PARAM_DELIM = '&'
  NAME_VAL_DELIM = '='
  SCHEME_DELIM = '://'
  UNIX_SLASH = '/'
  URL_ESCAPE_CHAR = '%'
  URL_ESCAPE_SEQ_LEN = 3 # expected length of URL espace sequences, aka len('%25')

  def __init__(self, left_url, right_url, names_only=False, hostnames=False,
      url_decode_params=False):
    """Initializes object and performs URL diffing."""
    self._left_url = self._normalize_url(left_url)
    self._right_url = self._normalize_url(right_url)
    self._names_only = names_only
    self._wants_hostname_diff = hostnames
    self._url_decode_params = url_decode_params
    self._diffs = []
    self._do_diff()

  def __str__(self):
    ret = []<commit_msg>convert left and right value init args to lists in ParamDiffEntry constructor. updated formatting logic to handle multiple values for a single key.<commit_after>#!/usr/bin/python2
"""
Copyright 2013 Google Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

"""A tool to show the differnce in URLs.

A tool for showing the differnces between URLs.  This is inspired by the unix
utility diff.
"""

import argparse
import copy
import logging
import sys

class Error(Exception):
  """Base exception class."""
  pass

class ParamDiffTypeError(Error):
  """Raised when an incorrect diff type used."""
  pass

class HostnameParseError(Error):
  """Raised when unable to parse hostname from URL."""
  pass

# TODO(macpd): investigate making this a namedtuple
class ParamDiffEntry(object):
  """Represents difference of 2 URL params with same name."""
  LEFT_ONLY = 1
  RIGHT_ONLY = 2
  BOTH_DIFFER = 3
  LEFT_HEADER_FORMAT = '{0}\n< {1}'
  LEFT_ADDITIONAL_SEPARATOR = '\n< '
  RIGHT_HEADER_FORMAT = '{0}\n> {1}'
  RIGHT_ADDITIONAL_SEPARATOR = '\n> '

  def __init__(self, name, left_value, right_value, diff_type):
    self._name = name
    self._left_val = list(left_value) if left_value else []
    self._right_val = list(right_value) if right_value else []
    try:
      if self._valid_diff_type(diff_type):
        self._type = diff_type
    except ParamDiffTypeError:
      logging.error("Incorrect diff type: %s", diff_type)
      self._type = self.BOTH_DIFFER

  def _valid_diff_type(self, diff_type):
    if (diff_type != self.LEFT_ONLY and diff_type != self.RIGHT_ONLY and
        diff_type != self.BOTH_DIFFER):
      raise ParamDiffTypeError('%s is not a valid diff type. ', diff_type)
    return True

  @property
  def name(self):
    return self._name

  def __str__(self):
    ret = self._name
    if self._type == self.LEFT_ONLY or self._type == self.BOTH_DIFFER:
      ret = self.LEFT_HEADER_FORMAT.format(ret, self.LEFT_ADDITIONAL_SEPARATOR.join(self._left_val))
    if self._type == self.RIGHT_ONLY or self._type == self.BOTH_DIFFER:
      ret = self.RIGHT_HEADER_FORMAT.format(ret, self.RIGHT_ADDITIONAL_SEPARATOR.join(self._right_val))
    return ret


class UrlDiffer(object):
  """Object to diff URLs.

  Diffs URLs upon intialization."""

  PATH_DELIM = '?'
  PARAM_DELIM = '&'
  NAME_VAL_DELIM = '='
  SCHEME_DELIM = '://'
  UNIX_SLASH = '/'
  URL_ESCAPE_CHAR = '%'
  URL_ESCAPE_SEQ_LEN = 3 # expected length of URL espace sequences, aka len('%25')

  def __init__(self, left_url, right_url, names_only=False, hostnames=False,
      url_decode_params=False):
    """Initializes object and performs URL diffing."""
    self._left_url = self._normalize_url(left_url)
    self._right_url = self._normalize_url(right_url)
    self._names_only = names_only
    self._wants_hostname_diff = hostnames
    self._url_decode_params = url_decode_params
    self._diffs = []
    self._do_diff()

  def __str__(self):
    ret = []
{'fraction_non_alphanumeric': 0.09701606732976281, 'fraction_numerical': 0.00612088752869166, 'mean_word_length': 3.8739746457867263, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2425292', 'n_tokens_mistral': 2147, 'n_tokens_neox': 2052, 'n_words': 644}