<commit_before><%
  import re
  import sys

  import markdown
  try:
      import pygments
      import pygments.formatters
      import pygments.lexers
      use_pygments = True
  except ImportError:
      use_pygments = False

  import pdoc

  # From language reference, but adds '.' to allow fully qualified names.
  pyident = re.compile('^[a-zA-Z_][a-zA-Z0-9_.]+$')
  indent = re.compile('^\s*')

  # Whether we're showing the module list or a single module.
  module_list = 'modules' in context.keys()

  def eprint(s):
      print >> sys.stderr, s

  def ident(s):
      return '<span class="ident">%s</span>' % s

  def sourceid(dobj):
      return 'source-%s' % dobj.refname

  def clean_source_lines(lines):
      """
      Cleans the source code so that pygments can render it well.

      Returns one string with all of the source code.
      """
      base_indent = len(indent.match(lines[0]).group(0))
      lines = [line[base_indent:] for line in lines]
      if not use_pygments:  # :-(
          return '<pre><code>%s</code></pre>' % (''.join(lines))

      pylex = pygments.lexers.PythonLexer()
      htmlform = pygments.formatters.HtmlFormatter(cssclass='codehilite')
      return pygments.highlight(''.join(lines), pylex, htmlform)

  def linkify(match):
      matched = match.group(0)
      ident = matched[1:-1]
      name, url = lookup(ident)
      if name is None:
          return matched
      return '[`%s`](%s)' % (name, url)

  def mark(s, linky=True):
      if linky:
        s, _ = re.subn('\b\n\b', ' ', s)
        if not module_list:
            s, _ = re.subn('`[^`]+`', linkify, s)
      
      extensions = []
      if use_pygments:
          extensions = ['codehilite(linenums=False)']
      s = markdown.markdown(s.strip(), extensions=extensions)
      return s

  def glimpse(s, length=100):
      if len(s) < length:
          return s
      return s[0:length] + '...'

  def module_url(m):
      """
      Returns a URL for `m`, which must be an instance of `Module`.
      Also, `m` must be a submodule of the module being documented.

      Namely, '.' import separators are replaced with '/' URL
      separators. Also, packages are translated as directories
      containing `index.html` corresponding to the `__init__` module,
      while modules are translated as regular HTML files with an
      `.m.html` suffix. (Given default values of 
      `pdoc.html_module_suffix` and `pdoc.html_package_name`.)
      """
      if module.name == m.name:
          return ''

      if len(link_prefix) > 0:
          base = m.name
      else:
          base = m.name[len(module.name)+1:]
      url = base.replace('.', '/')
      if m.is_package():
          url += '/%s' % pdoc.html_package_name
      else:
          url += pdoc.html_module_suffix
      return link_prefix + url

  def external_url(refname):
      """
      Attempts to guess an absolute URL for the external identifier
      given.

      Note that this just returns the refname with an ".ext" suffix.
      It will be up to whatever is interpreting the URLs to map it
      to an appropriate documentation page.
      """
      return '/%s.ext' % refname

  def is_external_linkable(name):
      return external_links and pyident.match(name) and '.' in name

  def lookup(refname):
      """
      Given a fully qualified identifier name, return its refname
      with respect to the current module and a value for a `href`
      attribute. If `refname` is not in the public interface of
      this module or its submodules, then `None` is returned for
      both return values. (Unless this module has enabled external
      linking.)

      In particular, this takes into account sub-modules and external
      identifiers. If `refname` is in the public API of the current
      module, then a local anchor link is given. If `refname` is in the
      public API of a sub-module, then a link to a different page with
      the appropriate anchor is given. Otherwise, `refname` is
      considered external and no link is used.
      """
      d = module.find_ident(refname)
      if isinstance(d, pdoc.External):
          if is_external_linkable(refname):
              return d.refname, external_url(d.refname)
          else:
              return None, None
      if isinstance(d, pdoc.Module):
          return d.refname, module_url(d)
      if module.is_public(d.refname):
          return d.name, '#%s' % d.refname
      return d.refname, '%s#%s' % (module_url(d.module), d.refname)

  def link(refname):
      """
      A convenience wrapper around `href` to produce the full
      `a` tag if `refname` is found. Otherwise, plain text of
      `refname` is returned.
      """
      name, url = lookup(refname)
      if name is None:
          return refname
      return '<a href="%s">%s</a>' % (url, name)
%>
<%def name="show_source(d)">
  % if show_source_code and d.source is not None and len(d.source) > 0:
    <p class="source_link"><a href="javascript:void(0);" onclick="toggle('${sourceid(d)}', this);">Show source.</a></p>
    <div id="${sourceid(d)}" class="source">
      ${clean_source_lines(d.source)}
    </div>
  % endif
</%def>

<%def name="show_desc(d, limit=None)">
  <%
    inherits = (hasattr(d, 'inherits')
                   and (len(d.docstring) == 0
                        or d.docstring == d.inherits.docstring))
    docstring = (d.inherits.docstring if inherits else d.docstring).strip()
    if limit is not None:
        docstring = glimpse(docstring, limit)
  %>
  % if len(docstring) > 0:
    % if inherits:
      <div class="desc inherited">${docstring | mark}</div>
    % else:
      <div class="desc">${docstring | mark}</div>
    % endif
  % endif
  <div class="source_cont">${show_source(d)}</div>
</%def>

<%def name="show_inheritance(d)">
  % if hasattr(d, 'inherits'):
      <p class="inheritance">
         <strong>Inheritance:</strong>
         % if hasattr(d.inherits, 'cls'):
           <code>${link(d.inherits.cls.refname)}</code>.<code>${link(d.inherits.refname)}</code>
         % else:
           <code>${link(d.inherits.refname)}</code>
         % endif
      </p>
  % endif
</%def>

<%def name="show_module_list(modules)">
<h1>Python module list</h1>
<commit_msg>Don't show source code of submodules ever.<commit_after><%
  import re
  import sys

  import markdown
  try:
      import pygments
      import pygments.formatters
      import pygments.lexers
      use_pygments = True
  except ImportError:
      use_pygments = False

  import pdoc

  # From language reference, but adds '.' to allow fully qualified names.
  pyident = re.compile('^[a-zA-Z_][a-zA-Z0-9_.]+$')
  indent = re.compile('^\s*')

  # Whether we're showing the module list or a single module.
  module_list = 'modules' in context.keys()

  def eprint(s):
      print >> sys.stderr, s

  def ident(s):
      return '<span class="ident">%s</span>' % s

  def sourceid(dobj):
      return 'source-%s' % dobj.refname

  def clean_source_lines(lines):
      """
      Cleans the source code so that pygments can render it well.

      Returns one string with all of the source code.
      """
      base_indent = len(indent.match(lines[0]).group(0))
      lines = [line[base_indent:] for line in lines]
      if not use_pygments:  # :-(
          return '<pre><code>%s</code></pre>' % (''.join(lines))

      pylex = pygments.lexers.PythonLexer()
      htmlform = pygments.formatters.HtmlFormatter(cssclass='codehilite')
      return pygments.highlight(''.join(lines), pylex, htmlform)

  def linkify(match):
      matched = match.group(0)
      ident = matched[1:-1]
      name, url = lookup(ident)
      if name is None:
          return matched
      return '[`%s`](%s)' % (name, url)

  def mark(s, linky=True):
      if linky:
        s, _ = re.subn('\b\n\b', ' ', s)
        if not module_list:
            s, _ = re.subn('`[^`]+`', linkify, s)
      
      extensions = []
      if use_pygments:
          extensions = ['codehilite(linenums=False)']
      s = markdown.markdown(s.strip(), extensions=extensions)
      return s

  def glimpse(s, length=100):
      if len(s) < length:
          return s
      return s[0:length] + '...'

  def module_url(m):
      """
      Returns a URL for `m`, which must be an instance of `Module`.
      Also, `m` must be a submodule of the module being documented.

      Namely, '.' import separators are replaced with '/' URL
      separators. Also, packages are translated as directories
      containing `index.html` corresponding to the `__init__` module,
      while modules are translated as regular HTML files with an
      `.m.html` suffix. (Given default values of 
      `pdoc.html_module_suffix` and `pdoc.html_package_name`.)
      """
      if module.name == m.name:
          return ''

      if len(link_prefix) > 0:
          base = m.name
      else:
          base = m.name[len(module.name)+1:]
      url = base.replace('.', '/')
      if m.is_package():
          url += '/%s' % pdoc.html_package_name
      else:
          url += pdoc.html_module_suffix
      return link_prefix + url

  def external_url(refname):
      """
      Attempts to guess an absolute URL for the external identifier
      given.

      Note that this just returns the refname with an ".ext" suffix.
      It will be up to whatever is interpreting the URLs to map it
      to an appropriate documentation page.
      """
      return '/%s.ext' % refname

  def is_external_linkable(name):
      return external_links and pyident.match(name) and '.' in name

  def lookup(refname):
      """
      Given a fully qualified identifier name, return its refname
      with respect to the current module and a value for a `href`
      attribute. If `refname` is not in the public interface of
      this module or its submodules, then `None` is returned for
      both return values. (Unless this module has enabled external
      linking.)

      In particular, this takes into account sub-modules and external
      identifiers. If `refname` is in the public API of the current
      module, then a local anchor link is given. If `refname` is in the
      public API of a sub-module, then a link to a different page with
      the appropriate anchor is given. Otherwise, `refname` is
      considered external and no link is used.
      """
      d = module.find_ident(refname)
      if isinstance(d, pdoc.External):
          if is_external_linkable(refname):
              return d.refname, external_url(d.refname)
          else:
              return None, None
      if isinstance(d, pdoc.Module):
          return d.refname, module_url(d)
      if module.is_public(d.refname):
          return d.name, '#%s' % d.refname
      return d.refname, '%s#%s' % (module_url(d.module), d.refname)

  def link(refname):
      """
      A convenience wrapper around `href` to produce the full
      `a` tag if `refname` is found. Otherwise, plain text of
      `refname` is returned.
      """
      name, url = lookup(refname)
      if name is None:
          return refname
      return '<a href="%s">%s</a>' % (url, name)
%>
<%def name="show_source(d)">
  % if show_source_code and d.source is not None and len(d.source) > 0:
    <p class="source_link"><a href="javascript:void(0);" onclick="toggle('${sourceid(d)}', this);">Show source.</a></p>
    <div id="${sourceid(d)}" class="source">
      ${clean_source_lines(d.source)}
    </div>
  % endif
</%def>

<%def name="show_desc(d, limit=None)">
  <%
    inherits = (hasattr(d, 'inherits')
                   and (len(d.docstring) == 0
                        or d.docstring == d.inherits.docstring))
    docstring = (d.inherits.docstring if inherits else d.docstring).strip()
    if limit is not None:
        docstring = glimpse(docstring, limit)
  %>
  % if len(docstring) > 0:
    % if inherits:
      <div class="desc inherited">${docstring | mark}</div>
    % else:
      <div class="desc">${docstring | mark}</div>
    % endif
  % endif
  % if not isinstance(d, pdoc.Module):
    <div class="source_cont">${show_source(d)}</div>
  % endif
</%def>

<%def name="show_inheritance(d)">
  % if hasattr(d, 'inherits'):
      <p class="inheritance">
         <strong>Inheritance:</strong>
         % if hasattr(d.inherits, 'cls'):
           <code>${link(d.inherits.cls.refname)}</code>.<code>${link(d.inherits.refname)}</code>
         % else:
           <code>${link(d.inherits.refname)}</code>
         % endif
      </p>
  % endif
</%def>

<%def name="show_module_list(modules)">
<h1>Python module list</h1>

{'fraction_non_alphanumeric': 0.12966804979253113, 'fraction_numerical': 0.003032237472071497, 'mean_word_length': 2.5047539149888145, 'pattern_counts': {'":': 0, '<': 87, '<?xml version=': 0, '>': 95, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3699998', 'n_tokens_mistral': 4051, 'n_tokens_neox': 3888, 'n_words': 1177}