<commit_before># $Id$

import sys, re, os, Cookie, errno
try:
  import cStringIO as StringIO
except ImportError:
  import StringIO

"""Object-oriented CGI interface."""


class Error(Exception):
  """The base class for all exceptions thrown by this module."""
  pass

class SequencingError(Error):
  """The exception thrown when functions are called out of order."""
  """
  For example, if you try to call a function altering the headers of your
  output when the headers have already been sent.
  """
  pass


_url_encre = re.compile(r"[^A-Za-z0-9_.!~*()-]") # RFC 2396 section 2.3
_url_decre = re.compile(r"%([0-9A-Fa-f]{2})")
_html_encre = re.compile("[&<>\"'+]")
# '+' is encoded because it is special in UTF-7, which the browser may select
# automatically if the content-type header does not specify the character
# encoding. This is paranoia and is not bulletproof, but it does no harm. See
# section 4 of www.microsoft.com/technet/security/topics/csoverv.asp
_html_encodes = { "&": "&amp;", "<": "&lt;", ">": "&gt;", "\"": "&quot;",
                  "'": "&#39;", "+": "&#43;" }

def html_encode(raw):
  """Return the string parameter HTML-encoded."""
  """
  Specifically, the following characters are encoded as entities:
    & < > " ' +
  """
  return re.sub(_html_encre, lambda m: _html_encodes[m.group(0)], str(raw))

def url_encode(raw):
  """Return the string parameter URL-encoded."""
  return re.sub(_url_encre, lambda m: "%%%02X" % ord(m.group(0)), str(raw))

def url_decode(enc):
  """Return the string parameter URL-decoded (including '+' -> ' ')."""
  s = enc.replace("+", " ")
  return re.sub(_url_decre, lambda m: chr(int(m.group(1), 16)), s)


__UNDEF__ = []

def _lookup(name, frame, locls):
  if name in locls:
    return "local", locls[name]
  if name in frame.f_globals:
    return "global", frame.f_globals[name]
  if "__builtins__" in frame.f_globals and \
    hasattr(frame.f_globals["__builtins__"], name):
    return "builtin", getattr(frame.f_globals["__builtins__"], name)
  return None, __UNDEF__


def _scanvars(reader, frame, locls):
  import tokenize, keyword
  vrs = []
  lasttoken = None
  parent = None
  prefix = ""
  for ttype, token, start, end, line in tokenize.generate_tokens(reader):
    if ttype == tokenize.NEWLINE:
      break
    elif ttype == tokenize.NAME and token not in keyword.kwlist:
      if lasttoken == ".":
        if parent is not __UNDEF__:
          value = getattr(parent, token, __UNDEF__)
          vrs.append((prefix + token, prefix, value))
      else:
        (where, value) = _lookup(token, frame, locls)
        vrs.append((token, where, value))
    elif token == ".":
      prefix += lasttoken + "."
      parent = value
    else:
      parent = None
      prefix = ""
    lasttoken = token
  return vrs


def _tb_encode(s):
  return html_encode(s).replace(" ", "&nbsp;")


def traceback(req, html=0):
  import traceback, time, types, linecache, inspect, repr
  repr = repr.Repr()
  repr.maxother = 80
  repr.maxstring = 80
  repr = repr.repr
  (etype, evalue, etb) = sys.exc_info()
  if type(etype) is types.ClassType:
    etype = etype.__name__
  if html:
    try:
      req.clear_headers()
      req.clear_output()
      req.set_header("Content-Type", "text/html; charset=iso-8859-1")
    except SequencingError:
      req.write("</font></font></font></script></object></blockquote></pre>"
        "</table></table></table></table></table></table></font></font>")
    req.write("""\
<?xml version="1.0" encoding="iso-8859-1"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><title>jonpy traceback: %s</title></head>
<body>
<style type="text/css"><!--
BODY { background-color: #f0f0f8; font-family: helveta, arial, sans-serif }
.tb_head { background-color: #6622aa; color: #ffffff }
.tb_title { font-size: x-large }
.tb_frame { background-color: #d8bbff }
.tb_lineno { font-size: smaller }
.tb_codehigh { background-color: #ffccee }
.tb_code { color: #909090 }
.tb_dump { color: #909090; font-size: smaller }
--></style>
<table width="100%%" cellspacing="0" cellpadding="0" border="0">
<tr class="tb_head">
<td valign="bottom" class="tb_title">&nbsp;<br /><strong>%s</strong></td>
<td align="right" valign="bottom">%s<br />%s</td></tr></table>
<p>A problem occurred in a Python script. Here is the sequence of
function calls leading up to the error, with the most recent first.</p>
""" % (_tb_encode(etype), _tb_encode(etype),
      "Python %s: %s" % (sys.version.split()[0], sys.executable),
      time.ctime(time.time())))
  req.error("jonpy error: %s at %s\n" % (etype, time.ctime(time.time())))
  # this code adapted from the standard cgitb module
  # unfortunately we cannot use that module directly,
  # mainly because it won't allow us to output to the log
  frames = []
  records = inspect.getinnerframes(etb, 7)
  records.reverse()
  for frame, fn, lnum, func, lines, index in records:
    if html:
      req.write("""\
<table width="100%" cellspacing="0" cellpadding="0" border="0">""")
    fn = fn and os.path.abspath(fn) or "?"<commit_msg>xhtml compliance for tracebacks<commit_after># $Id$

import sys, re, os, Cookie, errno
try:
  import cStringIO as StringIO
except ImportError:
  import StringIO

"""Object-oriented CGI interface."""


class Error(Exception):
  """The base class for all exceptions thrown by this module."""
  pass

class SequencingError(Error):
  """The exception thrown when functions are called out of order."""
  """
  For example, if you try to call a function altering the headers of your
  output when the headers have already been sent.
  """
  pass


_url_encre = re.compile(r"[^A-Za-z0-9_.!~*()-]") # RFC 2396 section 2.3
_url_decre = re.compile(r"%([0-9A-Fa-f]{2})")
_html_encre = re.compile("[&<>\"'+]")
# '+' is encoded because it is special in UTF-7, which the browser may select
# automatically if the content-type header does not specify the character
# encoding. This is paranoia and is not bulletproof, but it does no harm. See
# section 4 of www.microsoft.com/technet/security/topics/csoverv.asp
_html_encodes = { "&": "&amp;", "<": "&lt;", ">": "&gt;", "\"": "&quot;",
                  "'": "&#39;", "+": "&#43;" }

def html_encode(raw):
  """Return the string parameter HTML-encoded."""
  """
  Specifically, the following characters are encoded as entities:
    & < > " ' +
  """
  return re.sub(_html_encre, lambda m: _html_encodes[m.group(0)], str(raw))

def url_encode(raw):
  """Return the string parameter URL-encoded."""
  return re.sub(_url_encre, lambda m: "%%%02X" % ord(m.group(0)), str(raw))

def url_decode(enc):
  """Return the string parameter URL-decoded (including '+' -> ' ')."""
  s = enc.replace("+", " ")
  return re.sub(_url_decre, lambda m: chr(int(m.group(1), 16)), s)


__UNDEF__ = []

def _lookup(name, frame, locls):
  if name in locls:
    return "local", locls[name]
  if name in frame.f_globals:
    return "global", frame.f_globals[name]
  if "__builtins__" in frame.f_globals and \
    hasattr(frame.f_globals["__builtins__"], name):
    return "builtin", getattr(frame.f_globals["__builtins__"], name)
  return None, __UNDEF__


def _scanvars(reader, frame, locls):
  import tokenize, keyword
  vrs = []
  lasttoken = None
  parent = None
  prefix = ""
  for ttype, token, start, end, line in tokenize.generate_tokens(reader):
    if ttype == tokenize.NEWLINE:
      break
    elif ttype == tokenize.NAME and token not in keyword.kwlist:
      if lasttoken == ".":
        if parent is not __UNDEF__:
          value = getattr(parent, token, __UNDEF__)
          vrs.append((prefix + token, prefix, value))
      else:
        (where, value) = _lookup(token, frame, locls)
        vrs.append((token, where, value))
    elif token == ".":
      prefix += lasttoken + "."
      parent = value
    else:
      parent = None
      prefix = ""
    lasttoken = token
  return vrs


def _tb_encode(s):
  return html_encode(s).replace(" ", "&nbsp;")


def traceback(req, html=0):
  import traceback, time, types, linecache, inspect, repr
  repr = repr.Repr()
  repr.maxother = 80
  repr.maxstring = 80
  repr = repr.repr
  (etype, evalue, etb) = sys.exc_info()
  if type(etype) is types.ClassType:
    etype = etype.__name__
  if html:
    try:
      req.clear_headers()
      req.clear_output()
      req.set_header("Content-Type", "text/html; charset=iso-8859-1")
    except SequencingError:
      req.write("</font></font></font></script></object></blockquote></pre>"
        "</table></table></table></table></table></table></font></font>")
    req.write("""\
<?xml version="1.0" encoding="iso-8859-1"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><title>jonpy traceback: %s</title>
<style type="text/css"><!--
BODY { background-color: #f0f0f8; font-family: helveta, arial, sans-serif }
.tb_head { background-color: #6622aa; color: #ffffff }
.tb_title { font-size: x-large }
.tb_frame { background-color: #d8bbff }
.tb_lineno { font-size: smaller }
.tb_codehigh { background-color: #ffccee }
.tb_code { color: #909090 }
.tb_dump { color: #909090; font-size: smaller }
--></style>
</head><body>
<table width="100%%" cellspacing="0" cellpadding="0" border="0">
<tr class="tb_head">
<td valign="bottom" class="tb_title">&nbsp;<br /><strong>%s</strong></td>
<td align="right" valign="bottom">%s<br />%s</td></tr></table>
<p>A problem occurred in a Python script. Here is the sequence of
function calls leading up to the error, with the most recent first.</p>
""" % (_tb_encode(etype), _tb_encode(etype),
      "Python %s: %s" % (sys.version.split()[0], sys.executable),
      time.ctime(time.time())))
  req.error("jonpy error: %s at %s\n" % (etype, time.ctime(time.time())))
  # this code adapted from the standard cgitb module
  # unfortunately we cannot use that module directly,
  # mainly because it won't allow us to output to the log
  frames = []
  records = inspect.getinnerframes(etb, 7)
  records.reverse()
  for frame, fn, lnum, func, lines, index in records:
    if html:
      req.write("""\
<table width="100%" cellspacing="0" cellpadding="0" border="0">""")
    fn = fn and os.path.abspath(fn) or "?"
{'fraction_non_alphanumeric': 0.16920374707259953, 'fraction_numerical': 0.01639344262295082, 'mean_word_length': 3.7449074074074074, 'pattern_counts': {'":': 16, '<': 91, '<?xml version=': 2, '>': 93, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '851449', 'n_tokens_mistral': 3684, 'n_tokens_neox': 3502, 'n_words': 979}