<commit_before># pretty.py - Pretty formatting for db table rows. There are routines defined
# here for use in a terminal as well as on the web. 
#
# Copyright (c) 2013, <NAME>, all rights reserved.
# 
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#   * Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * The names of contributors may be used to endorse or promote products
#     derived from this software without specific prior written permission.
# 
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import sys, time, datetime
import json, re, string
from dateutil import tz

  ## Some JavaScripts that are embedded into some of the outputs. ##

js_confirmRemoveTerm = """
  function ConfirmRemoveTerm(id) {
    var r=window.confirm("Are you sure you want to delete term #" + id + "?");
    if (r==true) { 
      x=id; 
      var form = document.createElement("form");
      form.setAttribute("method", "post");
      form.setAttribute("action", "/term=" + id + "/remove");
      field = document.createElement("input");
      field.setAttribute("name", "id");
      field.setAttribute("value", id);
      form.appendChild(field);
      document.body.appendChild(form); 
      form.submit();
    } else { x="nil"; } } """

js_confirmRemoveComment = """
  function ConfirmRemoveComment(id) {
    var r=window.confirm("Are you sure you want to your comment?");
    if (r==true) { 
      x=id; 
      var form = document.createElement("form");
      form.setAttribute("method", "post");
      form.setAttribute("action", "/comment=" + id + "/remove");
      field = document.createElement("input");
      field.setAttribute("name", "id");
      field.setAttribute("value", id);
      form.appendChild(field);
      document.body.appendChild(form); 
      form.submit();
    } else { x="nil"; } } """

js_termAction = """
  function TermAction(id, v) {
    var form = document.createElement("form"); 
    form.setAttribute("method", "post");
    if (v == "up" || v == "down") {
      var action = "vote"; 
    } else {
      var action = "track"; 
    }
    form.setAttribute("action", "/term=" + id + "/" + action); 
    field = document.createElement("input"); 
    field.setAttribute("name", "action")
    field.setAttribute("value", v);
    form.appendChild(field);
    document.body.appendChild(form);
    form.submit(); 
  } """

js_copyToClipboard = """
  function CopyToClipboard(text) {
    window.prompt("Hit Ctrl-C (Cmd-C), then Enter to copy this reference to your clipboard. " +
                  "Embedding the reference in your term definition or comment " +
                  "will create a hyperlink to this term.", text);
  }
"""

#: Background color to display with term class. 
colorOf = { 'vernacular' : '#FFFF66', 
            'canonical' : '#3CEB10', 
            'deprecated' : '#E8E8E8' }

#: Name of months. See :func:`seaice.pretty.printPrettyDate`.
monthOf = [ 'January', 'February', 'March', 
            'April', 'May', 'June', 
            'July', 'August', 'September', 
            'October', 'November', 'December' ]
  
# A nice color: A5C6D6
tag_style = '''
style="font-size: 95%;
    font-family: 'Sans-Serif', Arial, serif;
    color:white; background-color:#0082C3;
    border-radius:4px; text-decoration:none"
'''

gtag_style = '''
style="font-size: 85%;
    font-family: 'Sans-Serif', Arial, serif;
    color:blue; background-color:#cceeff;
    border-radius:4px; text-decoration:none"
'''

ref_string = '<a href=/term={0} title="{2}">{1}</a>'
tag_string = '<a href=/tag/{0} ' + tag_style + '>&nbsp;<b>#</b>&nbsp;{1}&nbsp;</a>'
gtag_string = '<a href=/tag/{0} title="{2}" ' + gtag_style + '>&nbsp;{1}&nbsp;</a>'
term_tag_string = '<a href=/term={0} title="{1}">{2}</a>'

# Regular expressions for string matches.
token_ref_regex = re.compile("(?<!#\{g: )([#&]+)([\w.-]+)")
# Caution: exactly one space here -----^
# The "lookbehind" regex relies on _ref_norm using just one space.
# We use it to match #foo NOT inside a #{g:... construct.

#ref_regex = re.compile("#\{\s*(([gstkm])\s*:+)?\s*#*([^}|]*?)(\s*\|+\s*([^}]*?))?\s*\}")
ref_regex = re.compile("#\{\s*(([gstkm])\s*:+)?\s*([^}|]*?)(\s*\|+\s*([^}]*?))?\s*\}")
# subexpr start positions:    01                  2        3         4
#endrefs_regex = re.compile("#\{\s*([gve])\s*:\s*---\s*}\s*")
_xtag_regex = re.compile("#(([a-zA-Z][a-zA-Z0-9_\-\.]*)_term)")	# hack!
#tag_regex = re.compile("#([a-zA-Z][a-zA-Z0-9_\-\.]*[a-zA-Z0-9])")
_xterm_tag_regex = re.compile("#\{\s*([a-zA-Z0-9]+)\s*:\s*(related to[^\{\}]*)\}")	# hack!
#term_tag_regex = re.compile("#\{\s*([a-zA-Z0-9]+)\s*:\s*([^\{\}]*)\}")
permalink_regex = re.compile("^http://(.*)$")

def _token_ref_norm(m):
  """ Promote simple "#ref" into curly "#{t: ref} or, if ref
  begins with "#", into "#{g: ref}".

  :param string: The input string. 
  :returns: Modified plain text string.
  """

  #token = m.group(1)
  #if token.startswith('#'):
  #  return '#{g: ' + token + '}'
  sigil = m.group(1)
  token = m.group(2)
  if sigil == '#':
    return '#{g: #' + token + '}'
  elif sigil == '&':
    return '#{t: ' + token + '}'
  else:
    return sigil + token	# return untouched if doubled

def _xtag_norm(db_con, m):
  """ Promote old style "#xxx_term" into new style "#{g: #xxx | hNNNN }".
  """
  subtag = m.group(2)	# xxx hack, should match "ppsr" in "ppsr_term"
  return '#' + subtag
  #n, term = db_con.getTermByTermString(subtag)
  #if n == 1:
  #  term_string, concept_id = term['term_string'], term['concept_id']
  #elif n == 0:
  #  term_string, concept_id = (subtag + '(undefined)'), '-'
  #elif n == 2:
  #  term_string, concept_id = (subtag + '(ambiguous)'), '-'
  #return '#{g: #%s | %s }' % (term_string, concept_id)

def _xterm_tag_norm(db_con, m):
  """ Promote old style "#{hNNNN : related to}" into new style
  "#{t: term string | hNNNN }".
  """
  concept_id = m.group(1)
  term = db_con.getTermByConceptId(concept_id)
  if term:
    term_string = term['term_string']
  else:
    term_string = concept_id + '(undefined)'

  return '#{t: %s | %s }' % (term_string, concept_id)

def refs_norm(db_con, string, force=False): 
  """ Resolve references in text entries before storing in DB.

  :param db_con: DB connection.
  :type db_con: seaice.SeaIceConnector.SeaIceConnector
  :param string: The input string. <commit_msg>Many deferred messages below --- Per PPSR feedback, made search result looks like browse results Reference changes:   definition displayed when mouse hovers over term reference   tags can be defined like terms   as for #tag, one-word terms have shorthand reference notation (&term)   external link type added Identifier changes:   added production mode, so no real ARKs get minted during testing   prevents reminting ids on "import" when term already has an id (untested)   now binding kernel metadata on term creation and update Display changes:   newlines preserved on display (as <br>)   copy link feature folded into top level term string instance Documentation changes:   added a "guidelines" document   updated the "about" document<commit_after># pretty.py - Pretty formatting for db table rows. There are routines defined
# here for use in a terminal as well as on the web. 
#
# Copyright (c) 2013, <NAME>, all rights reserved.
# 
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#   * Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * The names of contributors may be used to endorse or promote products
#     derived from this software without specific prior written permission.
# 
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import sys, time, datetime
import json, re, string
from dateutil import tz

  ## Some JavaScripts that are embedded into some of the outputs. ##

js_confirmRemoveTerm = """
  function ConfirmRemoveTerm(id) {
    var r=window.confirm("Are you sure you want to delete term #" + id + "?");
    if (r==true) { 
      x=id; 
      var form = document.createElement("form");
      form.setAttribute("method", "post");
      form.setAttribute("action", "/term=" + id + "/remove");
      field = document.createElement("input");
      field.setAttribute("name", "id");
      field.setAttribute("value", id);
      form.appendChild(field);
      document.body.appendChild(form); 
      form.submit();
    } else { x="nil"; } } """

js_confirmRemoveComment = """
  function ConfirmRemoveComment(id) {
    var r=window.confirm("Are you sure you want to your comment?");
    if (r==true) { 
      x=id; 
      var form = document.createElement("form");
      form.setAttribute("method", "post");
      form.setAttribute("action", "/comment=" + id + "/remove");
      field = document.createElement("input");
      field.setAttribute("name", "id");
      field.setAttribute("value", id);
      form.appendChild(field);
      document.body.appendChild(form); 
      form.submit();
    } else { x="nil"; } } """

js_termAction = """
  function TermAction(id, v) {
    var form = document.createElement("form"); 
    form.setAttribute("method", "post");
    if (v == "up" || v == "down") {
      var action = "vote"; 
    } else {
      var action = "track"; 
    }
    form.setAttribute("action", "/term=" + id + "/" + action); 
    field = document.createElement("input"); 
    field.setAttribute("name", "action")
    field.setAttribute("value", v);
    form.appendChild(field);
    document.body.appendChild(form);
    form.submit(); 
  } """

js_copyToClipboard = """
  function CopyToClipboard(text) {
    window.prompt("Hit Ctrl-C (Cmd-C), then Enter to copy this reference to your clipboard. " +
                  "Embedding the reference in your term definition or comment " +
                  "will create a hyperlink to this term.", text);
  }
"""

#: Background color to display with term class. 
colorOf = { 'vernacular' : '#FFFF66', 
            'canonical' : '#3CEB10', 
            'deprecated' : '#E8E8E8' }

#: Name of months. See :func:`seaice.pretty.printPrettyDate`.
monthOf = [ 'January', 'February', 'March', 
            'April', 'May', 'June', 
            'July', 'August', 'September', 
            'October', 'November', 'December' ]
  
# A nice color: A5C6D6
tag_style = '''
style="font-size: 95%;
    font-family: 'Sans-Serif', Arial, serif;
    color:white; background-color:#0082C3;
    border-radius:4px; text-decoration:none"
'''

gtag_style = '''
style="font-size: 85%;
    font-family: 'Sans-Serif', Arial, serif;
    color:blue; background-color:#cceeff;
    border-radius:4px; text-decoration:none"
'''

ref_string = '<a href=/term={0} title="{2}">{1}</a>'
tag_string = '<a href=/tag/{0} ' + tag_style + '>&nbsp;<b>#</b>&nbsp;{1}&nbsp;</a>'
gtag_string = '<a href=/tag/{0} title="{2}" ' + gtag_style + '>&nbsp;{1}&nbsp;</a>'
term_tag_string = '<a href=/term={0} title="{1}">{2}</a>'

# Regular expressions for string matches.
token_ref_regex = re.compile("(?<!#\{g: )([#&]+)([\w.-]+)")
# Caution: exactly one space here -----^
# The "lookbehind" regex relies on _ref_norm using just one space.
# We use it to match #foo NOT inside a #{g:... construct.

#ref_regex = re.compile("#\{\s*(([gstkm])\s*:+)?\s*#*([^}|]*?)(\s*\|+\s*([^}]*?))?\s*\}")
ref_regex = re.compile("#\{\s*(([gstkm])\s*:+)?\s*([^}|]*?)(\s*\|+\s*([^}]*?))?\s*\}")
# subexpr start positions:    01                  2        3         4
#endrefs_regex = re.compile("#\{\s*([gve])\s*:\s*---\s*}\s*")
_xtag_regex = re.compile("#(([a-zA-Z][a-zA-Z0-9_\-\.]*)_term)")	# hack!
#tag_regex = re.compile("#([a-zA-Z][a-zA-Z0-9_\-\.]*[a-zA-Z0-9])")
_xterm_tag_regex = re.compile("#\{\s*([a-zA-Z0-9]+)\s*:\s*(related to[^\{\}]*)\}")	# hack!
#term_tag_regex = re.compile("#\{\s*([a-zA-Z0-9]+)\s*:\s*([^\{\}]*)\}")
permalink_regex = re.compile("^http://(.*)$")

def _token_ref_norm(m):
  """ Promote simple "#ref" into curly "#{t: ref} or, if ref
  begins with "#", into "#{g: ref}".

  :param string: The input string. 
  :returns: Modified plain text string.
  """

  #token = m.group(1)
  #if token.startswith('#'):
  #  return '#{g: ' + token + '}'
  sigil = m.group(1)
  token = m.group(2)
  if sigil == '#':
    return '#{g: #' + token + '}'
  elif sigil == '&':
    return '#{t: ' + token + '}'
  else:
    return sigil + token	# return untouched if doubled

def _xtag_norm(db_con, m):
  """ Promote old style "#xxx_term" into new style "#{g: #xxx | hNNNN }".
  """
  subtag = m.group(2)	# xxx hack, should match "ppsr" in "ppsr_term"
  return '#' + subtag
  #n, term = db_con.getTermByTermString(subtag)
  #if n == 1:
  #  term_string, concept_id = term['term_string'], term['concept_id']
  #elif n == 0:
  #  term_string, concept_id = (subtag + '(undefined)'), '-'
  #elif n == 2:
  #  term_string, concept_id = (subtag + '(ambiguous)'), '-'
  #return '#{g: #%s | %s }' % (term_string, concept_id)

def _xterm_tag_norm(db_con, m):
  """ Promote old style "#{hNNNN : related to}" into new style
  "#{t: term string | hNNNN }".
  """
  concept_id = m.group(1)
  term = db_con.getTermByConceptId(concept_id)
  if term:
    term_string = term['term_string']
  else:
    term_string = concept_id + '(undefined)'

  return '#{t: %s | %s }' % (term_string, concept_id)


def refs_norm(db_con, string, force=False): 
  """ Resolve references in text entries before storing in DB.

  :param db_con: DB connection.
  :type db_con: seaice.SeaIceConnector.SeaIceConnector
  :param string: The input string. 
{'fraction_non_alphanumeric': 0.16955150891191043, 'fraction_numerical': 0.007721510842288141, 'mean_word_length': 3.5631238990017615, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5348562', 'n_tokens_mistral': 5273, 'n_tokens_neox': 4989, 'n_words': 1499}